static_embeddings 0.1.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (32) hide show
  1. checksums.yaml +7 -0
  2. data/CHANGELOG.md +136 -0
  3. data/LICENSE.txt +21 -0
  4. data/README.md +439 -0
  5. data/docs/ARCHITECTURE.md +278 -0
  6. data/docs/MODEL_AUDIT.md +61 -0
  7. data/docs/PERFORMANCE.md +60 -0
  8. data/exe/static_embeddings +6 -0
  9. data/ext/static_embeddings/extconf.rb +44 -0
  10. data/ext/static_embeddings/se_embed.c +192 -0
  11. data/ext/static_embeddings/se_format.c +762 -0
  12. data/ext/static_embeddings/se_internal.h +266 -0
  13. data/ext/static_embeddings/se_tokenizer.c +506 -0
  14. data/ext/static_embeddings/se_unicode.c +120 -0
  15. data/ext/static_embeddings/static_embeddings.c +2070 -0
  16. data/lib/static_embeddings/cli.rb +178 -0
  17. data/lib/static_embeddings/converter.rb +284 -0
  18. data/lib/static_embeddings/errors.rb +6 -0
  19. data/lib/static_embeddings/format.rb +289 -0
  20. data/lib/static_embeddings/model.rb +48 -0
  21. data/lib/static_embeddings/paths.rb +29 -0
  22. data/lib/static_embeddings/reference.rb +191 -0
  23. data/lib/static_embeddings/safetensors.rb +87 -0
  24. data/lib/static_embeddings/unicode_tables.rb +127 -0
  25. data/lib/static_embeddings/version.rb +3 -0
  26. data/lib/static_embeddings.rb +118 -0
  27. data/static_embeddings.gemspec +45 -0
  28. data/tools/benchmark.rb +38 -0
  29. data/tools/build_demo_model.rb +16 -0
  30. data/tools/check_model2vec_parity.rb +107 -0
  31. data/tools/make_fixture_model.rb +165 -0
  32. metadata +134 -0
metadata ADDED
@@ -0,0 +1,134 @@
1
+ --- !ruby/object:Gem::Specification
2
+ name: static_embeddings
3
+ version: !ruby/object:Gem::Version
4
+ version: 0.1.1
5
+ platform: ruby
6
+ authors:
7
+ - Roman Haydarov
8
+ bindir: exe
9
+ cert_chain: []
10
+ date: 1980-01-02 00:00:00.000000000 Z
11
+ dependencies:
12
+ - !ruby/object:Gem::Dependency
13
+ name: bundler
14
+ requirement: !ruby/object:Gem::Requirement
15
+ requirements:
16
+ - - "~>"
17
+ - !ruby/object:Gem::Version
18
+ version: '2.0'
19
+ type: :development
20
+ prerelease: false
21
+ version_requirements: !ruby/object:Gem::Requirement
22
+ requirements:
23
+ - - "~>"
24
+ - !ruby/object:Gem::Version
25
+ version: '2.0'
26
+ - !ruby/object:Gem::Dependency
27
+ name: minitest
28
+ requirement: !ruby/object:Gem::Requirement
29
+ requirements:
30
+ - - "~>"
31
+ - !ruby/object:Gem::Version
32
+ version: '5.0'
33
+ type: :development
34
+ prerelease: false
35
+ version_requirements: !ruby/object:Gem::Requirement
36
+ requirements:
37
+ - - "~>"
38
+ - !ruby/object:Gem::Version
39
+ version: '5.0'
40
+ - !ruby/object:Gem::Dependency
41
+ name: rake
42
+ requirement: !ruby/object:Gem::Requirement
43
+ requirements:
44
+ - - "~>"
45
+ - !ruby/object:Gem::Version
46
+ version: '13.0'
47
+ type: :development
48
+ prerelease: false
49
+ version_requirements: !ruby/object:Gem::Requirement
50
+ requirements:
51
+ - - "~>"
52
+ - !ruby/object:Gem::Version
53
+ version: '13.0'
54
+ - !ruby/object:Gem::Dependency
55
+ name: rake-compiler
56
+ requirement: !ruby/object:Gem::Requirement
57
+ requirements:
58
+ - - "~>"
59
+ - !ruby/object:Gem::Version
60
+ version: '1.2'
61
+ type: :development
62
+ prerelease: false
63
+ version_requirements: !ruby/object:Gem::Requirement
64
+ requirements:
65
+ - - "~>"
66
+ - !ruby/object:Gem::Version
67
+ version: '1.2'
68
+ description: A small C-extension runtime for Model2Vec-style static embedding models.
69
+ Models are converted offline into a flat mmap-able .semb file; at runtime the gem
70
+ tokenizes (BERT WordPiece), looks up rows and mean-pools them. Releases the GVL
71
+ on large native work, rejects internal thread fan-out, and links nothing but libc.
72
+ email:
73
+ - romnhajdarov@gmail.com
74
+ executables:
75
+ - static_embeddings
76
+ extensions:
77
+ - ext/static_embeddings/extconf.rb
78
+ extra_rdoc_files: []
79
+ files:
80
+ - CHANGELOG.md
81
+ - LICENSE.txt
82
+ - README.md
83
+ - docs/ARCHITECTURE.md
84
+ - docs/MODEL_AUDIT.md
85
+ - docs/PERFORMANCE.md
86
+ - exe/static_embeddings
87
+ - ext/static_embeddings/extconf.rb
88
+ - ext/static_embeddings/se_embed.c
89
+ - ext/static_embeddings/se_format.c
90
+ - ext/static_embeddings/se_internal.h
91
+ - ext/static_embeddings/se_tokenizer.c
92
+ - ext/static_embeddings/se_unicode.c
93
+ - ext/static_embeddings/static_embeddings.c
94
+ - lib/static_embeddings.rb
95
+ - lib/static_embeddings/cli.rb
96
+ - lib/static_embeddings/converter.rb
97
+ - lib/static_embeddings/errors.rb
98
+ - lib/static_embeddings/format.rb
99
+ - lib/static_embeddings/model.rb
100
+ - lib/static_embeddings/paths.rb
101
+ - lib/static_embeddings/reference.rb
102
+ - lib/static_embeddings/safetensors.rb
103
+ - lib/static_embeddings/unicode_tables.rb
104
+ - lib/static_embeddings/version.rb
105
+ - static_embeddings.gemspec
106
+ - tools/benchmark.rb
107
+ - tools/build_demo_model.rb
108
+ - tools/check_model2vec_parity.rb
109
+ - tools/make_fixture_model.rb
110
+ homepage: https://github.com/roman-haidarov/static_embeddings
111
+ licenses:
112
+ - MIT
113
+ metadata:
114
+ homepage_uri: https://github.com/roman-haidarov/static_embeddings
115
+ source_code_uri: https://github.com/roman-haidarov/static_embeddings
116
+ changelog_uri: https://github.com/roman-haidarov/static_embeddings/blob/main/CHANGELOG.md
117
+ rdoc_options: []
118
+ require_paths:
119
+ - lib
120
+ required_ruby_version: !ruby/object:Gem::Requirement
121
+ requirements:
122
+ - - ">="
123
+ - !ruby/object:Gem::Version
124
+ version: 3.1.0
125
+ required_rubygems_version: !ruby/object:Gem::Requirement
126
+ requirements:
127
+ - - ">="
128
+ - !ruby/object:Gem::Version
129
+ version: '0'
130
+ requirements: []
131
+ rubygems_version: 3.6.7
132
+ specification_version: 4
133
+ summary: Fast local text embeddings for Ruby — no ONNX, no Rust, no network
134
+ test_files: []