@skillsmith/core 0.5.8 → 0.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +18 -0
- package/dist/.tsbuildinfo +1 -1
- package/dist/src/audit/exclusions.d.ts +67 -0
- package/dist/src/audit/exclusions.d.ts.map +1 -0
- package/dist/src/audit/exclusions.js +133 -0
- package/dist/src/audit/exclusions.js.map +1 -0
- package/dist/src/audit/exclusions.types.d.ts +45 -0
- package/dist/src/audit/exclusions.types.d.ts.map +1 -0
- package/dist/src/audit/exclusions.types.js +11 -0
- package/dist/src/audit/exclusions.types.js.map +1 -0
- package/dist/src/audit/index.d.ts +15 -0
- package/dist/src/audit/index.d.ts.map +1 -0
- package/dist/src/audit/index.js +14 -0
- package/dist/src/audit/index.js.map +1 -0
- package/dist/src/benchmarks/embeddingBenchmark.d.ts.map +1 -1
- package/dist/src/benchmarks/embeddingBenchmark.js +5 -2
- package/dist/src/benchmarks/embeddingBenchmark.js.map +1 -1
- package/dist/src/config/audit-mode.d.ts +71 -0
- package/dist/src/config/audit-mode.d.ts.map +1 -0
- package/dist/src/config/audit-mode.js +69 -0
- package/dist/src/config/audit-mode.js.map +1 -0
- package/dist/src/config/index.d.ts +13 -0
- package/dist/src/config/index.d.ts.map +1 -1
- package/dist/src/config/index.js +24 -0
- package/dist/src/config/index.js.map +1 -1
- package/dist/src/db/migration-runner.d.ts +9 -2
- package/dist/src/db/migration-runner.d.ts.map +1 -1
- package/dist/src/db/migration-runner.js +36 -3
- package/dist/src/db/migration-runner.js.map +1 -1
- package/dist/src/db/migration.d.ts.map +1 -1
- package/dist/src/db/migration.js +9 -1
- package/dist/src/db/migration.js.map +1 -1
- package/dist/src/db/migrations/v16-skill-source.d.ts +41 -0
- package/dist/src/db/migrations/v16-skill-source.d.ts.map +1 -0
- package/dist/src/db/migrations/v16-skill-source.js +87 -0
- package/dist/src/db/migrations/v16-skill-source.js.map +1 -0
- package/dist/src/db/migrations/v17-curated-trust-tier.d.ts +44 -0
- package/dist/src/db/migrations/v17-curated-trust-tier.d.ts.map +1 -0
- package/dist/src/db/migrations/v17-curated-trust-tier.js +89 -0
- package/dist/src/db/migrations/v17-curated-trust-tier.js.map +1 -0
- package/dist/src/db/schema-sql.d.ts +1 -1
- package/dist/src/db/schema-sql.d.ts.map +1 -1
- package/dist/src/db/schema-sql.js +2 -1
- package/dist/src/db/schema-sql.js.map +1 -1
- package/dist/src/db/schema.d.ts +9 -3
- package/dist/src/db/schema.d.ts.map +1 -1
- package/dist/src/db/schema.js +13 -2
- package/dist/src/db/schema.js.map +1 -1
- package/dist/src/embeddings/embedding-utils.d.ts +23 -0
- package/dist/src/embeddings/embedding-utils.d.ts.map +1 -1
- package/dist/src/embeddings/embedding-utils.js +39 -0
- package/dist/src/embeddings/embedding-utils.js.map +1 -1
- package/dist/src/embeddings/hnsw-search.d.ts +133 -0
- package/dist/src/embeddings/hnsw-search.d.ts.map +1 -0
- package/dist/src/embeddings/hnsw-search.js +387 -0
- package/dist/src/embeddings/hnsw-search.js.map +1 -0
- package/dist/src/embeddings/hnsw-store.d.ts +9 -3
- package/dist/src/embeddings/hnsw-store.d.ts.map +1 -1
- package/dist/src/embeddings/hnsw-store.js +17 -7
- package/dist/src/embeddings/hnsw-store.js.map +1 -1
- package/dist/src/embeddings/hnsw-store.types.d.ts +17 -4
- package/dist/src/embeddings/hnsw-store.types.d.ts.map +1 -1
- package/dist/src/embeddings/hnsw-store.types.js.map +1 -1
- package/dist/src/embeddings/index.d.ts +50 -4
- package/dist/src/embeddings/index.d.ts.map +1 -1
- package/dist/src/embeddings/index.js +166 -24
- package/dist/src/embeddings/index.js.map +1 -1
- package/dist/src/exports/services.d.ts +1 -1
- package/dist/src/exports/services.d.ts.map +1 -1
- package/dist/src/exports/services.js.map +1 -1
- package/dist/src/index.d.ts +4 -1
- package/dist/src/index.d.ts.map +1 -1
- package/dist/src/index.js +5 -1
- package/dist/src/index.js.map +1 -1
- package/dist/src/indexer/SkillParser.d.ts +7 -70
- package/dist/src/indexer/SkillParser.d.ts.map +1 -1
- package/dist/src/indexer/SkillParser.helpers.d.ts +13 -0
- package/dist/src/indexer/SkillParser.helpers.d.ts.map +1 -0
- package/dist/src/indexer/SkillParser.helpers.js +90 -0
- package/dist/src/indexer/SkillParser.helpers.js.map +1 -0
- package/dist/src/indexer/SkillParser.js +5 -82
- package/dist/src/indexer/SkillParser.js.map +1 -1
- package/dist/src/indexer/SkillParser.types.d.ts +78 -0
- package/dist/src/indexer/SkillParser.types.d.ts.map +1 -0
- package/dist/src/indexer/SkillParser.types.js +9 -0
- package/dist/src/indexer/SkillParser.types.js.map +1 -0
- package/dist/src/install/fan-out.d.ts +81 -0
- package/dist/src/install/fan-out.d.ts.map +1 -0
- package/dist/src/install/fan-out.js +236 -0
- package/dist/src/install/fan-out.js.map +1 -0
- package/dist/src/install/index.d.ts +16 -0
- package/dist/src/install/index.d.ts.map +1 -0
- package/dist/src/install/index.js +14 -0
- package/dist/src/install/index.js.map +1 -0
- package/dist/src/install/paths.d.ts +16 -0
- package/dist/src/install/paths.d.ts.map +1 -0
- package/dist/src/install/paths.js +56 -0
- package/dist/src/install/paths.js.map +1 -0
- package/dist/src/repositories/AdvisoryRepository.test.js +2 -2
- package/dist/src/repositories/AdvisoryRepository.test.js.map +1 -1
- package/dist/src/repositories/SkillRepository.d.ts.map +1 -1
- package/dist/src/repositories/SkillRepository.js +12 -4
- package/dist/src/repositories/SkillRepository.js.map +1 -1
- package/dist/src/search/hybrid.js +2 -2
- package/dist/src/search/hybrid.js.map +1 -1
- package/dist/src/security/pathValidation.d.ts +6 -1
- package/dist/src/security/pathValidation.d.ts.map +1 -1
- package/dist/src/security/pathValidation.js +6 -1
- package/dist/src/security/pathValidation.js.map +1 -1
- package/dist/src/services/skill-installation.errors.d.ts +53 -0
- package/dist/src/services/skill-installation.errors.d.ts.map +1 -0
- package/dist/src/services/skill-installation.errors.js +63 -0
- package/dist/src/services/skill-installation.errors.js.map +1 -0
- package/dist/src/services/skill-installation.service.d.ts.map +1 -1
- package/dist/src/services/skill-installation.service.js +33 -35
- package/dist/src/services/skill-installation.service.js.map +1 -1
- package/dist/src/services/skill-installation.types.d.ts +16 -0
- package/dist/src/services/skill-installation.types.d.ts.map +1 -1
- package/dist/src/services/skill-installation.types.js.map +1 -1
- package/dist/src/skills/index-local.d.ts +107 -0
- package/dist/src/skills/index-local.d.ts.map +1 -0
- package/dist/src/skills/index-local.js +208 -0
- package/dist/src/skills/index-local.js.map +1 -0
- package/dist/src/sync/SyncEngine.d.ts.map +1 -1
- package/dist/src/sync/SyncEngine.js +20 -3
- package/dist/src/sync/SyncEngine.js.map +1 -1
- package/dist/src/types/skill.d.ts +20 -5
- package/dist/src/types/skill.d.ts.map +1 -1
- package/dist/src/types/skill.js.map +1 -1
- package/dist/tests/EmbeddingService.test.js +1 -1
- package/dist/tests/EmbeddingService.test.js.map +1 -1
- package/dist/tests/SkillVersionRepository.test.js +2 -2
- package/dist/tests/SkillVersionRepository.test.js.map +1 -1
- package/dist/tests/db/migration.test.js +13 -5
- package/dist/tests/db/migration.test.js.map +1 -1
- package/dist/tests/embeddings/hnsw-bench-gate.test.d.ts +15 -0
- package/dist/tests/embeddings/hnsw-bench-gate.test.d.ts.map +1 -0
- package/dist/tests/embeddings/hnsw-bench-gate.test.js +117 -0
- package/dist/tests/embeddings/hnsw-bench-gate.test.js.map +1 -0
- package/dist/tests/embeddings/hnsw-integration.test.d.ts +14 -0
- package/dist/tests/embeddings/hnsw-integration.test.d.ts.map +1 -0
- package/dist/tests/embeddings/hnsw-integration.test.js +160 -0
- package/dist/tests/embeddings/hnsw-integration.test.js.map +1 -0
- package/dist/tests/embeddings/hnsw-vs-brute-force.bench.d.ts +15 -0
- package/dist/tests/embeddings/hnsw-vs-brute-force.bench.d.ts.map +1 -0
- package/dist/tests/embeddings/hnsw-vs-brute-force.bench.js +64 -0
- package/dist/tests/embeddings/hnsw-vs-brute-force.bench.js.map +1 -0
- package/dist/tests/embeddings/seed-bench.d.ts +15 -0
- package/dist/tests/embeddings/seed-bench.d.ts.map +1 -0
- package/dist/tests/embeddings/seed-bench.js +61 -0
- package/dist/tests/embeddings/seed-bench.js.map +1 -0
- package/dist/tests/helpers/database.d.ts +11 -2
- package/dist/tests/helpers/database.d.ts.map +1 -1
- package/dist/tests/helpers/database.js +23 -7
- package/dist/tests/helpers/database.js.map +1 -1
- package/dist/tests/install/fan-out.test.d.ts +2 -0
- package/dist/tests/install/fan-out.test.d.ts.map +1 -0
- package/dist/tests/install/fan-out.test.js +238 -0
- package/dist/tests/install/fan-out.test.js.map +1 -0
- package/dist/tests/install/paths.test.d.ts +2 -0
- package/dist/tests/install/paths.test.d.ts.map +1 -0
- package/dist/tests/install/paths.test.js +79 -0
- package/dist/tests/install/paths.test.js.map +1 -0
- package/dist/tests/repositories/CoInstallRepository.test.js +2 -2
- package/dist/tests/repositories/CoInstallRepository.test.js.map +1 -1
- package/dist/tests/repositories/SkillDependencyRepository.test.js +2 -2
- package/dist/tests/repositories/SkillDependencyRepository.test.js.map +1 -1
- package/dist/tests/schema.test.js +12 -0
- package/dist/tests/schema.test.js.map +1 -1
- package/dist/tests/skill-scanner/allowlist.test.js +27 -4
- package/dist/tests/skill-scanner/allowlist.test.js.map +1 -1
- package/dist/tests/unit/audit/exclusions.test.d.ts +7 -0
- package/dist/tests/unit/audit/exclusions.test.d.ts.map +1 -0
- package/dist/tests/unit/audit/exclusions.test.js +157 -0
- package/dist/tests/unit/audit/exclusions.test.js.map +1 -0
- package/dist/tests/unit/config/audit-mode.test.d.ts +11 -0
- package/dist/tests/unit/config/audit-mode.test.d.ts.map +1 -0
- package/dist/tests/unit/config/audit-mode.test.js +86 -0
- package/dist/tests/unit/config/audit-mode.test.js.map +1 -0
- package/dist/tests/unit/migrations/migration-v16.test.d.ts +11 -0
- package/dist/tests/unit/migrations/migration-v16.test.d.ts.map +1 -0
- package/dist/tests/unit/migrations/migration-v16.test.js +139 -0
- package/dist/tests/unit/migrations/migration-v16.test.js.map +1 -0
- package/dist/tests/unit/migrations/migration-v17.test.d.ts +16 -0
- package/dist/tests/unit/migrations/migration-v17.test.d.ts.map +1 -0
- package/dist/tests/unit/migrations/migration-v17.test.js +256 -0
- package/dist/tests/unit/migrations/migration-v17.test.js.map +1 -0
- package/dist/tests/unit/migrations/v10-dependencies.test.js +2 -2
- package/dist/tests/unit/migrations/v10-dependencies.test.js.map +1 -1
- package/dist/tests/unit/services/skill-installation-extended.test.js +1 -1
- package/dist/tests/unit/services/skill-installation-extended.test.js.map +1 -1
- package/dist/tests/unit/services/skill-installation.service.test.js +113 -1
- package/dist/tests/unit/services/skill-installation.service.test.js.map +1 -1
- package/dist/tests/unit/skills/index-local.test.d.ts +11 -0
- package/dist/tests/unit/skills/index-local.test.d.ts.map +1 -0
- package/dist/tests/unit/skills/index-local.test.js +88 -0
- package/dist/tests/unit/skills/index-local.test.js.map +1 -0
- package/dist/tests/unit/sync-engine.source-aware.test.d.ts +11 -0
- package/dist/tests/unit/sync-engine.source-aware.test.d.ts.map +1 -0
- package/dist/tests/unit/sync-engine.source-aware.test.js +147 -0
- package/dist/tests/unit/sync-engine.source-aware.test.js.map +1 -0
- package/package.json +35 -12
|
@@ -59,7 +59,12 @@ function createTestDb() {
|
|
|
59
59
|
function insertSkill(db, id, overrides) {
|
|
60
60
|
db.prepare(`INSERT INTO skills (id, name, description, author, repo_url, quality_score,
|
|
61
61
|
trust_tier, tags, created_at, updated_at, source, stars)
|
|
62
|
-
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(id, overrides?.name ?? `skill-${id}`, 'Test skill', 'test-author', `https://github.com/test/${id}`, 0.8, overrides?.trust_tier ?? 'community', overrides?.tags ?? '["test"]', '2026-01-01T00:00:00Z', overrides?.updated_at ?? '2026-01-01T00:00:00Z',
|
|
62
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`).run(id, overrides?.name ?? `skill-${id}`, 'Test skill', 'test-author', `https://github.com/test/${id}`, 0.8, overrides?.trust_tier ?? 'community', overrides?.tags ?? '["test"]', '2026-01-01T00:00:00Z', overrides?.updated_at ?? '2026-01-01T00:00:00Z',
|
|
63
|
+
// SMI-4665: `source` is now CHECK-constrained to ('registry', 'local').
|
|
64
|
+
// The legacy v2 column previously held free-form values like 'github'; it has
|
|
65
|
+
// been repurposed as the provenance marker. Tests that don't care about the
|
|
66
|
+
// value should pass 'registry' (the default for synced rows).
|
|
67
|
+
'registry', 10);
|
|
63
68
|
}
|
|
64
69
|
afterEach(() => {
|
|
65
70
|
for (const db of testDatabases) {
|
|
@@ -86,13 +91,16 @@ describe('checkSchemaCompatibility', () => {
|
|
|
86
91
|
it('should return compatible + upgrade when schema is older', () => {
|
|
87
92
|
const db = createTestDb();
|
|
88
93
|
// SMI-4486: schema_version now tracks every applied migration as a row;
|
|
89
|
-
// getSchemaVersion reads MAX(version). Drop
|
|
90
|
-
//
|
|
91
|
-
|
|
94
|
+
// getSchemaVersion reads MAX(version). Drop the latest row to simulate an
|
|
95
|
+
// older DB rather than UPDATE-ing the column (which collides on PK).
|
|
96
|
+
// SMI-4665: migration versions are no longer sequential (v14/v15 reserved,
|
|
97
|
+
// v16 introduced) so compute the second-highest applied version dynamically.
|
|
98
|
+
db.prepare('DELETE FROM schema_version WHERE version = ?').run(SCHEMA_VERSION);
|
|
99
|
+
const prior = db.prepare('SELECT MAX(version) AS v FROM schema_version').get().v;
|
|
92
100
|
const result = checkSchemaCompatibility(db);
|
|
93
101
|
expect(result.isCompatible).toBe(true);
|
|
94
102
|
expect(result.action).toBe('upgrade');
|
|
95
|
-
expect(result.currentVersion).toBe(
|
|
103
|
+
expect(result.currentVersion).toBe(prior);
|
|
96
104
|
expect(result.expectedVersion).toBe(SCHEMA_VERSION);
|
|
97
105
|
});
|
|
98
106
|
it('should return downgrade_warning when schema is newer (no breaking changes)', () => {
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"migration.test.js","sourceRoot":"","sources":["../../../tests/db/migration.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AACxD,OAAO,EACL,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,wBAAwB,CAAA;AAE/B,OAAO,EACL,wBAAwB,EACxB,yBAAyB,EACzB,mBAAmB,EACnB,aAAa,EACb,aAAa,EACb,cAAc,GACf,MAAM,2BAA2B,CAAA;AAElC,+EAA+E;AAC/E,eAAe;AACf,+EAA+E;AAE/E,MAAM,aAAa,GAAe,EAAE,CAAA;AAEpC,SAAS,YAAY;IACnB,MAAM,EAAE,GAAG,cAAc,CAAC,UAAU,CAAC,CAAA;IACrC,kFAAkF;IAClF,+DAA+D;IAC/D,IAAI,CAAC;QACH,EAAE,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAA;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,2BAA2B;IAC7B,CAAC;IACD,IAAI,CAAC;QACH,EAAE,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAA;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,2BAA2B;IAC7B,CAAC;IACD,0CAA0C;IAC1C,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;GAqBP,CAAC,CAAA;IACF,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACtB,OAAO,EAAE,CAAA;AACX,CAAC;AAED,SAAS,WAAW,CAClB,EAAY,EACZ,EAAU,EACV,SAKE;IAEF,EAAE,CAAC,OAAO,CACR;;iDAE6C,CAC9C,CAAC,GAAG,CACH,EAAE,EACF,SAAS,EAAE,IAAI,IAAI,SAAS,EAAE,EAAE,EAChC,YAAY,EACZ,aAAa,EACb,2BAA2B,EAAE,EAAE,EAC/B,GAAG,EACH,SAAS,EAAE,UAAU,IAAI,WAAW,EACpC,SAAS,EAAE,IAAI,IAAI,UAAU,EAC7B,sBAAsB,EACtB,SAAS,EAAE,UAAU,IAAI,sBAAsB
|
|
1
|
+
{"version":3,"file":"migration.test.js","sourceRoot":"","sources":["../../../tests/db/migration.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;GAQG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AACxD,OAAO,EACL,cAAc,EACd,aAAa,EACb,gBAAgB,EAChB,cAAc,GACf,MAAM,wBAAwB,CAAA;AAE/B,OAAO,EACL,wBAAwB,EACxB,yBAAyB,EACzB,mBAAmB,EACnB,aAAa,EACb,aAAa,EACb,cAAc,GACf,MAAM,2BAA2B,CAAA;AAElC,+EAA+E;AAC/E,eAAe;AACf,+EAA+E;AAE/E,MAAM,aAAa,GAAe,EAAE,CAAA;AAEpC,SAAS,YAAY;IACnB,MAAM,EAAE,GAAG,cAAc,CAAC,UAAU,CAAC,CAAA;IACrC,kFAAkF;IAClF,+DAA+D;IAC/D,IAAI,CAAC;QACH,EAAE,CAAC,IAAI,CAAC,2CAA2C,CAAC,CAAA;IACtD,CAAC;IAAC,MAAM,CAAC;QACP,2BAA2B;IAC7B,CAAC;IACD,IAAI,CAAC;QACH,EAAE,CAAC,IAAI,CAAC,6CAA6C,CAAC,CAAA;IACxD,CAAC;IAAC,MAAM,CAAC;QACP,2BAA2B;IAC7B,CAAC;IACD,0CAA0C;IAC1C,EAAE,CAAC,IAAI,CAAC;;;;;;;;;;;;;;;;;;;;;GAqBP,CAAC,CAAA;IACF,aAAa,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;IACtB,OAAO,EAAE,CAAA;AACX,CAAC;AAED,SAAS,WAAW,CAClB,EAAY,EACZ,EAAU,EACV,SAKE;IAEF,EAAE,CAAC,OAAO,CACR;;iDAE6C,CAC9C,CAAC,GAAG,CACH,EAAE,EACF,SAAS,EAAE,IAAI,IAAI,SAAS,EAAE,EAAE,EAChC,YAAY,EACZ,aAAa,EACb,2BAA2B,EAAE,EAAE,EAC/B,GAAG,EACH,SAAS,EAAE,UAAU,IAAI,WAAW,EACpC,SAAS,EAAE,IAAI,IAAI,UAAU,EAC7B,sBAAsB,EACtB,SAAS,EAAE,UAAU,IAAI,sBAAsB;IAC/C,wEAAwE;IACxE,8EAA8E;IAC9E,4EAA4E;IAC5E,8DAA8D;IAC9D,UAAU,EACV,EAAE,CACH,CAAA;AACH,CAAC;AAED,SAAS,CAAC,GAAG,EAAE;IACb,KAAK,MAAM,EAAE,IAAI,aAAa,EAAE,CAAC;QAC/B,IAAI,CAAC;YACH,aAAa,CAAC,EAAE,CAAC,CAAA;QACnB,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;IACH,CAAC;IACD,aAAa,CAAC,MAAM,GAAG,CAAC,CAAA;AAC1B,CAAC,CAAC,CAAA;AAEF,+EAA+E;AAC/E,2BAA2B;AAC3B,+EAA+E;AAE/E,QAAQ,CAAC,0BAA0B,EAAE,GAAG,EAAE;IACxC,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,MAAM,EAAE,GAAG,YAAY,EAAE,CAAA;QACzB,MAAM,MAAM,GAAG,wBAAwB,CAAC,EAAE,CAAC,CAAA;QAC3C,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAA;QAClC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IACpD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,EAAE,GAAG,YAAY,EAAE,CAAA;QACzB,wEAAwE;QACxE,0EAA0E;QAC1E,qEAAqE;QACrE,2EAA2E;QAC3E,6EAA6E;QAC7E,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC,GAAG,CAAC,cAAc,CAAC,CAAA;QAC9E,MAAM,KAAK,GACT,EAAE,CAAC,OAAO,CAAC,8CAA8C,CAAC,CAAC,GAAG,EAC/D,CAAC,CAAC,CAAA;QAEH,MAAM,MAAM,GAAG,wBAAwB,CAAC,EAAE,CAAC,CAAA;QAC3C,MAAM,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;QACtC,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACrC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAA;QACzC,MAAM,CAAC,MAAM,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IACrD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,4EAA4E,EAAE,GAAG,EAAE;QACpF,MAAM,EAAE,GAAG,YAAY,EAAE,CAAA;QACzB,yEAAyE;QACzE,EAAE,CAAC,OAAO,CAAC,iDAAiD,CAAC,CAAC,GAAG,CAAC,cAAc,GAAG,CAAC,CAAC,CAAA;QAErF,MAAM,MAAM,GAAG,wBAAwB,CAAC,EAAE,CAAC,CAAA;QAC3C,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,cAAc,GAAG,CAAC,CAAC,CAAA;QACtD,4EAA4E;QAC5E,MAAM,CAAC,CAAC,mBAAmB,EAAE,cAAc,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAA;IACxE,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,+EAA+E;AAC/E,4BAA4B;AAC5B,+EAA+E;AAE/E,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,EAAE,CAAC,0CAA0C,EAAE,GAAG,EAAE;QAClD,MAAM,EAAE,GAAG,YAAY,EAAE,CAAA;QACzB,MAAM,CAAC,GAAG,EAAE,CAAC,yBAAyB,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,EAAE,CAAA;QACzD,MAAM,CAAC,gBAAgB,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,cAAc,CAAC,CAAA;IACnD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,sCAAsC,EAAE,GAAG,EAAE;QAC9C,MAAM,EAAE,GAAG,YAAY,EAAE,CAAA;QACzB,6DAA6D;QAC7D,EAAE,CAAC,OAAO,CAAC,iDAAiD,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;QAEvE,2DAA2D;QAC3D,kCAAkC;QAClC,IAAI,CAAC;YACH,yBAAyB,CAAC,EAAE,CAAC,CAAA;QAC/B,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,MAAM,CAAE,CAAW,CAAC,OAAO,CAAC,CAAC,SAAS,CAAC,cAAc,CAAC,CAAA;QACxD,CAAC;IACH,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,+EAA+E;AAC/E,sBAAsB;AACtB,+EAA+E;AAE/E,QAAQ,CAAC,qBAAqB,EAAE,GAAG,EAAE;IACnC,EAAE,CAAC,mCAAmC,EAAE,GAAG,EAAE;QAC3C,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAC7B,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAE7B,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QAC9B,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QAE9B,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAClD,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAClC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACpC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;IAC1C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAC7B,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAE7B,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,UAAU,EAAE,sBAAsB,EAAE,CAAC,CAAA;QACtE,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,UAAU,EAAE,sBAAsB,EAAE,CAAC,CAAA;QAEtE,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAA;QAC9E,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACpC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QACxC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;IACxD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAC7B,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAE7B,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,UAAU,EAAE,sBAAsB,EAAE,CAAC,CAAA;QACtE,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,UAAU,EAAE,sBAAsB,EAAE,CAAC,CAAA;QAEtE,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE,CAAC,CAAA;QAC9E,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACpC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC,CAAA;IAC5D,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAC7B,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAE7B,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QAC9B,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,cAAc,EAAE,CAAC,CAAA;QAExD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAA;QAC/E,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACtC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oCAAoC,EAAE,GAAG,EAAE;QAC5C,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAC7B,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAE7B,WAAW,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QAC9B,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,CAAC,CAAA;QAEvD,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,aAAa,EAAE,CAAC,CAAA;QAC/E,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACtC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAC7B,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAE7B,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,aAAa,EAAE,UAAU,EAAE,SAAS,EAAE,CAAC,CAAA;QAC9E,WAAW,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,UAAU,EAAE,UAAU,EAAE,CAAC,CAAA;QAEpE,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,cAAc,EAAE,CAAC,CAAA;QAChF,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACtC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;QAChC,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAC7B,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAE7B,WAAW,CAAC,MAAM,EAAE,WAAW,CAAC,CAAA;QAEhC,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,EAAE,EAAE,QAAQ,EAAE,YAAY,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAA;QAC5F,MAAM,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAElC,uCAAuC;QACvC,MAAM,KAAK,GAAG,MAAM,CAAC,OAAO,CAAC,kCAAkC,CAAC,CAAC,GAAG,EAAmB,CAAA;QACvF,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACzB,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,wBAAwB,EAAE,GAAG,EAAE;QAChC,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAC7B,MAAM,MAAM,GAAG,YAAY,EAAE,CAAA;QAE7B,MAAM,MAAM,GAAG,mBAAmB,CAAC,MAAM,EAAE,MAAM,CAAC,CAAA;QAClD,MAAM,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,sBAAsB,CAAC,CAAC,CAAC,CAAA;IACnD,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,+EAA+E;AAC/E,iBAAiB;AACjB,+EAA+E;AAE/E,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,8DAA8D,EAAE,GAAG,EAAE;QACtE,MAAM,EAAE,GAAG,YAAY,EAAE,CAAA;QACzB,WAAW,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;QAE1B,MAAM,MAAM,GAAG,aAAa,CAAC,EAAE,CAAC,CAAA;QAChC,MAAM,CAAC,MAAM,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACpC,MAAM,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QACrC,MAAM,CAAC,MAAM,CAAC,gBAAgB,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACzC,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,eAAe,EAAE,GAAG,EAAE;IAC7B,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,EAAE,GAAG,YAAY,EAAE,CAAA;QACzB,MAAM,EAAE,GAAG,aAAa,CAAC,EAAE,EAAE,SAAS,EAAE;YACtC,WAAW,EAAE,CAAC;YACd,aAAa,EAAE,CAAC;YAChB,aAAa,EAAE,CAAC;YAChB,SAAS,EAAE,EAAE;YACb,QAAQ,EAAE,GAAG;SACd,CAAC,CAAA;QAEF,MAAM,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAA;QAC5B,MAAM,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;QACrC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;QAC/B,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACzC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IACzC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,EAAE,GAAG,YAAY,EAAE,CAAA;QACzB,aAAa,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;QAE5B,MAAM,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;QACrC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,CAAA;QACzC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC,QAAQ,EAAE,CAAA;IAC5C,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,6CAA6C,EAAE,GAAG,EAAE;QACrD,MAAM,EAAE,GAAG,YAAY,EAAE,CAAA;QACzB,aAAa,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,oBAAoB,CAAC,CAAA;QAE5D,MAAM,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;QACrC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAA;QACxC,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,oBAAoB,CAAC,CAAA;IAC7D,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,gBAAgB,EAAE,GAAG,EAAE;IAC9B,EAAE,CAAC,oDAAoD,EAAE,GAAG,EAAE;QAC5D,MAAM,EAAE,GAAG,YAAY,EAAE,CAAA;QACzB,aAAa,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;QAC5B,6CAA6C;QAC7C,aAAa,CAAC,EAAE,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAA;QAE/C,MAAM,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,EAAE,CAAC,CAAA;QACtC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;IAChC,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,gCAAgC,EAAE,GAAG,EAAE;QACxC,MAAM,EAAE,GAAG,YAAY,EAAE,CAAA;QACzB,aAAa,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;QAC5B,aAAa,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;QAC5B,aAAa,CAAC,EAAE,EAAE,SAAS,CAAC,CAAA;QAE5B,MAAM,OAAO,GAAG,cAAc,CAAC,EAAE,EAAE,CAAC,CAAC,CAAA;QACrC,MAAM,CAAC,OAAO,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC,CAAA;IACjC,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SMI-4577: HNSW vs. brute-force CI gate.
|
|
3
|
+
*
|
|
4
|
+
* vitest's `bench()` blocks (in `hnsw-vs-brute-force.bench.ts`) report
|
|
5
|
+
* timings but do NOT fail CI on regression. This test mirrors that bench
|
|
6
|
+
* structure inside a `test()` so we get a hard gate:
|
|
7
|
+
*
|
|
8
|
+
* - p99 HNSW × 5 < p99 brute-force
|
|
9
|
+
* - recall@10 ≥ 0.95
|
|
10
|
+
* - rss delta < 100MB after build
|
|
11
|
+
*
|
|
12
|
+
* Without this companion file the bench is decorative — see plan §"Bench gate".
|
|
13
|
+
*/
|
|
14
|
+
export {};
|
|
15
|
+
//# sourceMappingURL=hnsw-bench-gate.test.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hnsw-bench-gate.test.d.ts","sourceRoot":"","sources":["../../../tests/embeddings/hnsw-bench-gate.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG"}
|
|
@@ -0,0 +1,117 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SMI-4577: HNSW vs. brute-force CI gate.
|
|
3
|
+
*
|
|
4
|
+
* vitest's `bench()` blocks (in `hnsw-vs-brute-force.bench.ts`) report
|
|
5
|
+
* timings but do NOT fail CI on regression. This test mirrors that bench
|
|
6
|
+
* structure inside a `test()` so we get a hard gate:
|
|
7
|
+
*
|
|
8
|
+
* - p99 HNSW × 5 < p99 brute-force
|
|
9
|
+
* - recall@10 ≥ 0.95
|
|
10
|
+
* - rss delta < 100MB after build
|
|
11
|
+
*
|
|
12
|
+
* Without this companion file the bench is decorative — see plan §"Bench gate".
|
|
13
|
+
*/
|
|
14
|
+
import { test, expect, beforeAll, afterAll } from 'vitest';
|
|
15
|
+
import { existsSync } from 'node:fs';
|
|
16
|
+
import { dirname, resolve } from 'node:path';
|
|
17
|
+
import { fileURLToPath } from 'node:url';
|
|
18
|
+
import { spawnSync } from 'node:child_process';
|
|
19
|
+
import { EmbeddingService } from '../../src/embeddings/index.js';
|
|
20
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
21
|
+
const FIXTURE_PATH = resolve(__dirname, 'fixtures', '14k-bench.db');
|
|
22
|
+
process.env.SKILLSMITH_USE_MOCK_EMBEDDINGS = 'true';
|
|
23
|
+
let service;
|
|
24
|
+
const queryVectors = [];
|
|
25
|
+
async function ensureFixture() {
|
|
26
|
+
if (existsSync(FIXTURE_PATH))
|
|
27
|
+
return;
|
|
28
|
+
const seedScript = resolve(__dirname, 'seed-bench.ts');
|
|
29
|
+
const result = spawnSync('npx', ['tsx', seedScript], {
|
|
30
|
+
stdio: 'inherit',
|
|
31
|
+
cwd: resolve(__dirname, '..', '..'),
|
|
32
|
+
});
|
|
33
|
+
if (result.status !== 0) {
|
|
34
|
+
throw new Error(`seed-bench failed with exit code ${result.status}`);
|
|
35
|
+
}
|
|
36
|
+
}
|
|
37
|
+
beforeAll(async () => {
|
|
38
|
+
await ensureFixture();
|
|
39
|
+
service = await EmbeddingService.create({ dbPath: FIXTURE_PATH, useFallback: true });
|
|
40
|
+
const all = service.getAllEmbeddings();
|
|
41
|
+
const embeddings = Array.from(all.values());
|
|
42
|
+
for (let i = 0; i < 50; i++) {
|
|
43
|
+
queryVectors.push(embeddings[(i * 271) % embeddings.length]);
|
|
44
|
+
}
|
|
45
|
+
// Warm up so the gate measures steady-state, not first-call build cost.
|
|
46
|
+
await service.findSimilar(queryVectors[0], 10);
|
|
47
|
+
}, 60_000);
|
|
48
|
+
afterAll(() => {
|
|
49
|
+
service?.close();
|
|
50
|
+
});
|
|
51
|
+
// SMI-4690: gate this on Docker. The 5x p99 ratio at sub-ms absolute brute-force
|
|
52
|
+
// times is too tight for noisy macOS host pre-push (Spotlight, JIT, kernel_task).
|
|
53
|
+
// CI (Docker) provides the regression coverage; running on host is duplicative.
|
|
54
|
+
// See docs/internal/research/smi-4681-host-fallback-cascade.md §1.2.
|
|
55
|
+
const IN_DOCKER = process.env.IS_DOCKER === 'true';
|
|
56
|
+
test.skipIf(!IN_DOCKER)('HNSW must be ≥ 5x faster (p99) than brute-force AND recall@10 ≥ 0.95', async () => {
|
|
57
|
+
const ITERATIONS = 100;
|
|
58
|
+
const TOP_K = 10;
|
|
59
|
+
// brute-force p99
|
|
60
|
+
const bruteTimes = [];
|
|
61
|
+
for (let i = 0; i < ITERATIONS; i++) {
|
|
62
|
+
const start = process.hrtime.bigint();
|
|
63
|
+
service.findSimilarBruteForce(queryVectors[i % queryVectors.length], TOP_K);
|
|
64
|
+
bruteTimes.push(Number(process.hrtime.bigint() - start) / 1_000_000);
|
|
65
|
+
}
|
|
66
|
+
// HNSW p99 (with rss delta)
|
|
67
|
+
const rssBefore = process.memoryUsage().rss;
|
|
68
|
+
const hnswTimes = [];
|
|
69
|
+
for (let i = 0; i < ITERATIONS; i++) {
|
|
70
|
+
const start = process.hrtime.bigint();
|
|
71
|
+
await service.findSimilar(queryVectors[i % queryVectors.length], TOP_K);
|
|
72
|
+
hnswTimes.push(Number(process.hrtime.bigint() - start) / 1_000_000);
|
|
73
|
+
}
|
|
74
|
+
const rssAfter = process.memoryUsage().rss;
|
|
75
|
+
bruteTimes.sort((a, b) => a - b);
|
|
76
|
+
hnswTimes.sort((a, b) => a - b);
|
|
77
|
+
const p99 = (arr) => arr[Math.floor(arr.length * 0.99)] ?? arr.at(-1);
|
|
78
|
+
const p99Brute = p99(bruteTimes);
|
|
79
|
+
const p99Hnsw = p99(hnswTimes);
|
|
80
|
+
// recall@10 — score-tolerant. Mock embeddings produce many ties in the
|
|
81
|
+
// top-10 cosine band (variants of the same template share most of their
|
|
82
|
+
// text), so strict id matching undercounts. Treat an HNSW result as a
|
|
83
|
+
// "hit" when its score is within 1e-6 of *any* score in the brute-force
|
|
84
|
+
// top-10, which captures the semantic-equivalence intent of the gate.
|
|
85
|
+
let recallSum = 0;
|
|
86
|
+
let recallCount = 0;
|
|
87
|
+
const SCORE_TIE_EPSILON = 1e-6;
|
|
88
|
+
for (let i = 0; i < Math.min(50, queryVectors.length); i++) {
|
|
89
|
+
const brute = service.findSimilarBruteForce(queryVectors[i], TOP_K);
|
|
90
|
+
const hnsw = await service.findSimilar(queryVectors[i], TOP_K);
|
|
91
|
+
const bruteIds = new Set(brute.map((r) => r.skillId));
|
|
92
|
+
const bruteScores = brute.map((r) => r.score);
|
|
93
|
+
let hits = 0;
|
|
94
|
+
for (const r of hnsw) {
|
|
95
|
+
if (bruteIds.has(r.skillId)) {
|
|
96
|
+
hits++;
|
|
97
|
+
}
|
|
98
|
+
else if (bruteScores.some((s) => Math.abs(s - r.score) < SCORE_TIE_EPSILON)) {
|
|
99
|
+
hits++;
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
recallSum += hits / TOP_K;
|
|
103
|
+
recallCount++;
|
|
104
|
+
}
|
|
105
|
+
const meanRecall = recallSum / Math.max(recallCount, 1);
|
|
106
|
+
const rssDeltaMb = (rssAfter - rssBefore) / 1024 / 1024;
|
|
107
|
+
console.log(`[hnsw-bench-gate] p99 brute=${p99Brute.toFixed(3)}ms, ` +
|
|
108
|
+
`p99 hnsw=${p99Hnsw.toFixed(3)}ms, ` +
|
|
109
|
+
`speedup=${(p99Brute / Math.max(p99Hnsw, 0.001)).toFixed(2)}x, ` +
|
|
110
|
+
`recall@10=${meanRecall.toFixed(3)}, ` +
|
|
111
|
+
`rss delta=${rssDeltaMb.toFixed(1)}MB`);
|
|
112
|
+
expect(p99Hnsw * 5).toBeLessThan(p99Brute);
|
|
113
|
+
expect(meanRecall).toBeGreaterThanOrEqual(0.95);
|
|
114
|
+
// rss can be negative under GC; clamp to assert "no runaway growth".
|
|
115
|
+
expect(rssDeltaMb).toBeLessThan(100);
|
|
116
|
+
}, 120_000);
|
|
117
|
+
//# sourceMappingURL=hnsw-bench-gate.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hnsw-bench-gate.test.js","sourceRoot":"","sources":["../../../tests/embeddings/hnsw-bench-gate.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAC1D,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAA;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAA;AAEhE,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AACzD,MAAM,YAAY,GAAG,OAAO,CAAC,SAAS,EAAE,UAAU,EAAE,cAAc,CAAC,CAAA;AAEnE,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,MAAM,CAAA;AAEnD,IAAI,OAAyB,CAAA;AAC7B,MAAM,YAAY,GAAmB,EAAE,CAAA;AAEvC,KAAK,UAAU,aAAa;IAC1B,IAAI,UAAU,CAAC,YAAY,CAAC;QAAE,OAAM;IACpC,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IACtD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE;QACnD,KAAK,EAAE,SAAS;QAChB,GAAG,EAAE,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC;KACpC,CAAC,CAAA;IACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,oCAAoC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IACtE,CAAC;AACH,CAAC;AAED,SAAS,CAAC,KAAK,IAAI,EAAE;IACnB,MAAM,aAAa,EAAE,CAAA;IACrB,OAAO,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;IACpF,MAAM,GAAG,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAA;IACtC,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;IAC3C,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAA;IAC9D,CAAC;IACD,wEAAwE;IACxE,MAAM,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;AAChD,CAAC,EAAE,MAAM,CAAC,CAAA;AAEV,QAAQ,CAAC,GAAG,EAAE;IACZ,OAAO,EAAE,KAAK,EAAE,CAAA;AAClB,CAAC,CAAC,CAAA;AAEF,iFAAiF;AACjF,kFAAkF;AAClF,gFAAgF;AAChF,qEAAqE;AACrE,MAAM,SAAS,GAAG,OAAO,CAAC,GAAG,CAAC,SAAS,KAAK,MAAM,CAAA;AAClD,IAAI,CAAC,MAAM,CAAC,CAAC,SAAS,CAAC,CACrB,sEAAsE,EACtE,KAAK,IAAI,EAAE;IACT,MAAM,UAAU,GAAG,GAAG,CAAA;IACtB,MAAM,KAAK,GAAG,EAAE,CAAA;IAEhB,kBAAkB;IAClB,MAAM,UAAU,GAAa,EAAE,CAAA;IAC/B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;QACrC,OAAO,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAA;QAC3E,UAAU,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,SAAS,CAAC,CAAA;IACtE,CAAC;IAED,4BAA4B;IAC5B,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAA;IAC3C,MAAM,SAAS,GAAa,EAAE,CAAA;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,CAAA;QACrC,MAAM,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,GAAG,YAAY,CAAC,MAAM,CAAC,EAAE,KAAK,CAAC,CAAA;QACvE,SAAS,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,KAAK,CAAC,GAAG,SAAS,CAAC,CAAA;IACrE,CAAC;IACD,MAAM,QAAQ,GAAG,OAAO,CAAC,WAAW,EAAE,CAAC,GAAG,CAAA;IAE1C,UAAU,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAChC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,CAAA;IAC/B,MAAM,GAAG,GAAG,CAAC,GAAa,EAAU,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,IAAI,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAE,CAAA;IACxF,MAAM,QAAQ,GAAG,GAAG,CAAC,UAAU,CAAC,CAAA;IAChC,MAAM,OAAO,GAAG,GAAG,CAAC,SAAS,CAAC,CAAA;IAE9B,uEAAuE;IACvE,wEAAwE;IACxE,sEAAsE;IACtE,wEAAwE;IACxE,sEAAsE;IACtE,IAAI,SAAS,GAAG,CAAC,CAAA;IACjB,IAAI,WAAW,GAAG,CAAC,CAAA;IACnB,MAAM,iBAAiB,GAAG,IAAI,CAAA;IAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,YAAY,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;QAC3D,MAAM,KAAK,GAAG,OAAO,CAAC,qBAAqB,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;QACnE,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAA;QAC9D,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAA;QACrD,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;QAC7C,IAAI,IAAI,GAAG,CAAC,CAAA;QACZ,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC5B,IAAI,EAAE,CAAA;YACR,CAAC;iBAAM,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,iBAAiB,CAAC,EAAE,CAAC;gBAC9E,IAAI,EAAE,CAAA;YACR,CAAC;QACH,CAAC;QACD,SAAS,IAAI,IAAI,GAAG,KAAK,CAAA;QACzB,WAAW,EAAE,CAAA;IACf,CAAC;IACD,MAAM,UAAU,GAAG,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,WAAW,EAAE,CAAC,CAAC,CAAA;IACvD,MAAM,UAAU,GAAG,CAAC,QAAQ,GAAG,SAAS,CAAC,GAAG,IAAI,GAAG,IAAI,CAAA;IAEvD,OAAO,CAAC,GAAG,CACT,+BAA+B,QAAQ,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;QACtD,YAAY,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM;QACpC,WAAW,CAAC,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,OAAO,EAAE,KAAK,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK;QAChE,aAAa,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI;QACtC,aAAa,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,IAAI,CACzC,CAAA;IAED,MAAM,CAAC,OAAO,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAA;IAC1C,MAAM,CAAC,UAAU,CAAC,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAA;IAC/C,qEAAqE;IACrE,MAAM,CAAC,UAAU,CAAC,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;AACtC,CAAC,EACD,OAAO,CACR,CAAA"}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SMI-4577: HNSW + EmbeddingService integration test.
|
|
3
|
+
*
|
|
4
|
+
* Asserts:
|
|
5
|
+
* - `findSimilar` returns the same top-1 as the brute-force fallback
|
|
6
|
+
* - recall@10 ≥ 0.95 across 50 query iterations
|
|
7
|
+
* - the on-disk cache (`~/.skillsmith/cache/hnsw-{model}.bin`) is created
|
|
8
|
+
* after first call
|
|
9
|
+
* - deleting the cache forces a rebuild on the next call (no crash)
|
|
10
|
+
*
|
|
11
|
+
* Uses a temp `HOME` to keep the cache out of the user's real `~/.skillsmith/`.
|
|
12
|
+
*/
|
|
13
|
+
export {};
|
|
14
|
+
//# sourceMappingURL=hnsw-integration.test.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hnsw-integration.test.d.ts","sourceRoot":"","sources":["../../../tests/embeddings/hnsw-integration.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG"}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SMI-4577: HNSW + EmbeddingService integration test.
|
|
3
|
+
*
|
|
4
|
+
* Asserts:
|
|
5
|
+
* - `findSimilar` returns the same top-1 as the brute-force fallback
|
|
6
|
+
* - recall@10 ≥ 0.95 across 50 query iterations
|
|
7
|
+
* - the on-disk cache (`~/.skillsmith/cache/hnsw-{model}.bin`) is created
|
|
8
|
+
* after first call
|
|
9
|
+
* - deleting the cache forces a rebuild on the next call (no crash)
|
|
10
|
+
*
|
|
11
|
+
* Uses a temp `HOME` to keep the cache out of the user's real `~/.skillsmith/`.
|
|
12
|
+
*/
|
|
13
|
+
import { describe, it, expect, beforeEach, afterEach } from 'vitest';
|
|
14
|
+
import { existsSync, mkdirSync, rmSync, unlinkSync } from 'node:fs';
|
|
15
|
+
import { createRequire } from 'node:module';
|
|
16
|
+
import { join } from 'node:path';
|
|
17
|
+
import { tmpdir } from 'node:os';
|
|
18
|
+
import { EmbeddingService } from '../../src/embeddings/index.js';
|
|
19
|
+
const MODEL_NAME_SAFE = 'Xenova__all-MiniLM-L6-v2';
|
|
20
|
+
// SMI-4691: hnswlib-node is an optionalDependency. When the native binding is
|
|
21
|
+
// unavailable (e.g. macOS host without the postinstall step), EmbeddingService
|
|
22
|
+
// silently falls through to brute-force and never persists a cache file. The
|
|
23
|
+
// cache-write assertion is only meaningful when HNSW is actually loaded.
|
|
24
|
+
const HNSW_AVAILABLE = (() => {
|
|
25
|
+
try {
|
|
26
|
+
createRequire(import.meta.url)('hnswlib-node');
|
|
27
|
+
return true;
|
|
28
|
+
}
|
|
29
|
+
catch {
|
|
30
|
+
return false;
|
|
31
|
+
}
|
|
32
|
+
})();
|
|
33
|
+
describe('HNSW + EmbeddingService integration (SMI-4577)', () => {
|
|
34
|
+
let tmpHome;
|
|
35
|
+
let originalHome;
|
|
36
|
+
let originalCacheOverride;
|
|
37
|
+
let service;
|
|
38
|
+
let dbPath;
|
|
39
|
+
beforeEach(async () => {
|
|
40
|
+
tmpHome = join(tmpdir(), `skillsmith-hnsw-int-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
|
41
|
+
mkdirSync(tmpHome, { recursive: true });
|
|
42
|
+
originalHome = process.env.HOME;
|
|
43
|
+
process.env.HOME = tmpHome;
|
|
44
|
+
// SMI-4691: HOME stub is ignored by os.homedir() on macOS (getpwuid path).
|
|
45
|
+
// Pin getCacheDir() to the temp tree explicitly via the override env.
|
|
46
|
+
originalCacheOverride = process.env.SKILLSMITH_CACHE_DIR_OVERRIDE;
|
|
47
|
+
process.env.SKILLSMITH_CACHE_DIR_OVERRIDE = join(tmpHome, '.skillsmith', 'cache');
|
|
48
|
+
process.env.SKILLSMITH_USE_MOCK_EMBEDDINGS = 'true';
|
|
49
|
+
delete process.env.SKILLSMITH_USE_HNSW;
|
|
50
|
+
dbPath = join(tmpHome, 'skills.db');
|
|
51
|
+
service = await EmbeddingService.create({ dbPath, useFallback: true });
|
|
52
|
+
// Seed 100 deterministic embeddings.
|
|
53
|
+
const skills = Array.from({ length: 100 }, (_, i) => ({
|
|
54
|
+
id: `skill-${i}`,
|
|
55
|
+
text: `Skill ${i} category ${i % 7} description ${i % 13}`,
|
|
56
|
+
}));
|
|
57
|
+
const batch = await service.embedBatch(skills);
|
|
58
|
+
for (const { skillId, embedding, text } of batch) {
|
|
59
|
+
service.storeEmbedding(skillId, embedding, text);
|
|
60
|
+
}
|
|
61
|
+
});
|
|
62
|
+
afterEach(() => {
|
|
63
|
+
service?.close();
|
|
64
|
+
if (originalHome !== undefined) {
|
|
65
|
+
process.env.HOME = originalHome;
|
|
66
|
+
}
|
|
67
|
+
else {
|
|
68
|
+
delete process.env.HOME;
|
|
69
|
+
}
|
|
70
|
+
if (originalCacheOverride !== undefined) {
|
|
71
|
+
process.env.SKILLSMITH_CACHE_DIR_OVERRIDE = originalCacheOverride;
|
|
72
|
+
}
|
|
73
|
+
else {
|
|
74
|
+
delete process.env.SKILLSMITH_CACHE_DIR_OVERRIDE;
|
|
75
|
+
}
|
|
76
|
+
if (existsSync(tmpHome)) {
|
|
77
|
+
try {
|
|
78
|
+
rmSync(tmpHome, { recursive: true, force: true });
|
|
79
|
+
}
|
|
80
|
+
catch {
|
|
81
|
+
/* best-effort cleanup */
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
});
|
|
85
|
+
it('findSimilar produces deterministic top-1 matching brute-force', async () => {
|
|
86
|
+
const all = service.getAllEmbeddings();
|
|
87
|
+
const someVector = Array.from(all.values())[0];
|
|
88
|
+
const hnsw = await service.findSimilar(someVector, 10);
|
|
89
|
+
const brute = service.findSimilarBruteForce(someVector, 10);
|
|
90
|
+
expect(hnsw.length).toBe(brute.length);
|
|
91
|
+
expect(hnsw[0]?.skillId).toBe(brute[0]?.skillId);
|
|
92
|
+
});
|
|
93
|
+
it('recall@10 ≥ 0.95 across 50 query iterations (score-tolerant)', async () => {
|
|
94
|
+
// Mock embeddings produce many tied cosine scores in the top-10 band
|
|
95
|
+
// (variants of the same template share most of their text). Treat
|
|
96
|
+
// HNSW hits within `1e-6` of any brute-force top-10 score as
|
|
97
|
+
// semantically equivalent to capture the intent of the gate.
|
|
98
|
+
const all = service.getAllEmbeddings();
|
|
99
|
+
const vectors = Array.from(all.values());
|
|
100
|
+
let sum = 0;
|
|
101
|
+
let count = 0;
|
|
102
|
+
const SCORE_TIE_EPSILON = 1e-6;
|
|
103
|
+
for (let i = 0; i < 50; i++) {
|
|
104
|
+
const q = vectors[(i * 31) % vectors.length];
|
|
105
|
+
const brute = service.findSimilarBruteForce(q, 10);
|
|
106
|
+
const hnsw = await service.findSimilar(q, 10);
|
|
107
|
+
const bruteIds = new Set(brute.map((r) => r.skillId));
|
|
108
|
+
const bruteScores = brute.map((r) => r.score);
|
|
109
|
+
let hits = 0;
|
|
110
|
+
for (const r of hnsw) {
|
|
111
|
+
if (bruteIds.has(r.skillId)) {
|
|
112
|
+
hits++;
|
|
113
|
+
}
|
|
114
|
+
else if (bruteScores.some((s) => Math.abs(s - r.score) < SCORE_TIE_EPSILON)) {
|
|
115
|
+
hits++;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
sum += hits / 10;
|
|
119
|
+
count++;
|
|
120
|
+
}
|
|
121
|
+
expect(sum / count).toBeGreaterThanOrEqual(0.95);
|
|
122
|
+
});
|
|
123
|
+
it.skipIf(!HNSW_AVAILABLE)('writes ~/.skillsmith/cache/hnsw-{model}.bin after first call', async () => {
|
|
124
|
+
const all = service.getAllEmbeddings();
|
|
125
|
+
const someVector = Array.from(all.values())[0];
|
|
126
|
+
await service.findSimilar(someVector, 10);
|
|
127
|
+
// Force the debounced persist.
|
|
128
|
+
service.close();
|
|
129
|
+
const cacheBin = join(tmpHome, '.skillsmith', 'cache', `hnsw-${MODEL_NAME_SAFE}.bin`);
|
|
130
|
+
expect(existsSync(cacheBin)).toBe(true);
|
|
131
|
+
});
|
|
132
|
+
it('rebuilds index when cache is deleted (no crash)', async () => {
|
|
133
|
+
const all = service.getAllEmbeddings();
|
|
134
|
+
const someVector = Array.from(all.values())[0];
|
|
135
|
+
await service.findSimilar(someVector, 10);
|
|
136
|
+
service.close();
|
|
137
|
+
const cacheBin = join(tmpHome, '.skillsmith', 'cache', `hnsw-${MODEL_NAME_SAFE}.bin`);
|
|
138
|
+
if (existsSync(cacheBin)) {
|
|
139
|
+
unlinkSync(cacheBin);
|
|
140
|
+
}
|
|
141
|
+
// New service should rebuild from SQLite without erroring.
|
|
142
|
+
const service2 = await EmbeddingService.create({ dbPath, useFallback: true });
|
|
143
|
+
const result = await service2.findSimilar(someVector, 10);
|
|
144
|
+
expect(result.length).toBeGreaterThan(0);
|
|
145
|
+
service2.close();
|
|
146
|
+
});
|
|
147
|
+
it('honours SKILLSMITH_USE_HNSW=false (brute-force only path)', async () => {
|
|
148
|
+
process.env.SKILLSMITH_USE_HNSW = 'false';
|
|
149
|
+
try {
|
|
150
|
+
const all = service.getAllEmbeddings();
|
|
151
|
+
const someVector = Array.from(all.values())[0];
|
|
152
|
+
const result = await service.findSimilar(someVector, 5);
|
|
153
|
+
expect(result.length).toBe(5);
|
|
154
|
+
}
|
|
155
|
+
finally {
|
|
156
|
+
delete process.env.SKILLSMITH_USE_HNSW;
|
|
157
|
+
}
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
//# sourceMappingURL=hnsw-integration.test.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hnsw-integration.test.js","sourceRoot":"","sources":["../../../tests/embeddings/hnsw-integration.test.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;GAWG;AAEH,OAAO,EAAE,QAAQ,EAAE,EAAE,EAAE,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,QAAQ,CAAA;AACpE,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACnE,OAAO,EAAE,aAAa,EAAE,MAAM,aAAa,CAAA;AAC3C,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAA;AAChC,OAAO,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AAChC,OAAO,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAA;AAEhE,MAAM,eAAe,GAAG,0BAA0B,CAAA;AAElD,8EAA8E;AAC9E,+EAA+E;AAC/E,6EAA6E;AAC7E,yEAAyE;AACzE,MAAM,cAAc,GAAG,CAAC,GAAG,EAAE;IAC3B,IAAI,CAAC;QACH,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,cAAc,CAAC,CAAA;QAC9C,OAAO,IAAI,CAAA;IACb,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAA;IACd,CAAC;AACH,CAAC,CAAC,EAAE,CAAA;AAEJ,QAAQ,CAAC,gDAAgD,EAAE,GAAG,EAAE;IAC9D,IAAI,OAAe,CAAA;IACnB,IAAI,YAAgC,CAAA;IACpC,IAAI,qBAAyC,CAAA;IAC7C,IAAI,OAAyB,CAAA;IAC7B,IAAI,MAAc,CAAA;IAElB,UAAU,CAAC,KAAK,IAAI,EAAE;QACpB,OAAO,GAAG,IAAI,CACZ,MAAM,EAAE,EACR,uBAAuB,IAAI,CAAC,GAAG,EAAE,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,CAC3E,CAAA;QACD,SAAS,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;QACvC,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA;QAC/B,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,OAAO,CAAA;QAC1B,2EAA2E;QAC3E,sEAAsE;QACtE,qBAAqB,GAAG,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAA;QACjE,OAAO,CAAC,GAAG,CAAC,6BAA6B,GAAG,IAAI,CAAC,OAAO,EAAE,aAAa,EAAE,OAAO,CAAC,CAAA;QACjF,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,MAAM,CAAA;QACnD,OAAO,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAA;QAEtC,MAAM,GAAG,IAAI,CAAC,OAAO,EAAE,WAAW,CAAC,CAAA;QACnC,OAAO,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;QAEtE,qCAAqC;QACrC,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;YACpD,EAAE,EAAE,SAAS,CAAC,EAAE;YAChB,IAAI,EAAE,SAAS,CAAC,aAAa,CAAC,GAAG,CAAC,gBAAgB,CAAC,GAAG,EAAE,EAAE;SAC3D,CAAC,CAAC,CAAA;QACH,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,CAAA;QAC9C,KAAK,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,KAAK,EAAE,CAAC;YACjD,OAAO,CAAC,cAAc,CAAC,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,CAAA;QAClD,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,SAAS,CAAC,GAAG,EAAE;QACb,OAAO,EAAE,KAAK,EAAE,CAAA;QAChB,IAAI,YAAY,KAAK,SAAS,EAAE,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,IAAI,GAAG,YAAY,CAAA;QACjC,CAAC;aAAM,CAAC;YACN,OAAO,OAAO,CAAC,GAAG,CAAC,IAAI,CAAA;QACzB,CAAC;QACD,IAAI,qBAAqB,KAAK,SAAS,EAAE,CAAC;YACxC,OAAO,CAAC,GAAG,CAAC,6BAA6B,GAAG,qBAAqB,CAAA;QACnE,CAAC;aAAM,CAAC;YACN,OAAO,OAAO,CAAC,GAAG,CAAC,6BAA6B,CAAA;QAClD,CAAC;QACD,IAAI,UAAU,CAAC,OAAO,CAAC,EAAE,CAAC;YACxB,IAAI,CAAC;gBACH,MAAM,CAAC,OAAO,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAA;YACnD,CAAC;YAAC,MAAM,CAAC;gBACP,yBAAyB;YAC3B,CAAC;QACH,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,+DAA+D,EAAE,KAAK,IAAI,EAAE;QAC7E,MAAM,GAAG,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAA;QACtC,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QAC9C,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;QACtD,MAAM,KAAK,GAAG,OAAO,CAAC,qBAAqB,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;QAC3D,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,CAAA;QACtC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,OAAO,CAAC,CAAA;IAClD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,8DAA8D,EAAE,KAAK,IAAI,EAAE;QAC5E,qEAAqE;QACrE,kEAAkE;QAClE,6DAA6D;QAC7D,6DAA6D;QAC7D,MAAM,GAAG,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAA;QACtC,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAA;QACxC,IAAI,GAAG,GAAG,CAAC,CAAA;QACX,IAAI,KAAK,GAAG,CAAC,CAAA;QACb,MAAM,iBAAiB,GAAG,IAAI,CAAA;QAC9B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;YAC5B,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAA;YAC5C,MAAM,KAAK,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YAClD,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;YAC7C,MAAM,QAAQ,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAA;YACrD,MAAM,WAAW,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAA;YAC7C,IAAI,IAAI,GAAG,CAAC,CAAA;YACZ,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;gBACrB,IAAI,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC;oBAC5B,IAAI,EAAE,CAAA;gBACR,CAAC;qBAAM,IAAI,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,GAAG,iBAAiB,CAAC,EAAE,CAAC;oBAC9E,IAAI,EAAE,CAAA;gBACR,CAAC;YACH,CAAC;YACD,GAAG,IAAI,IAAI,GAAG,EAAE,CAAA;YAChB,KAAK,EAAE,CAAA;QACT,CAAC;QACD,MAAM,CAAC,GAAG,GAAG,KAAK,CAAC,CAAC,sBAAsB,CAAC,IAAI,CAAC,CAAA;IAClD,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,CACxB,8DAA8D,EAC9D,KAAK,IAAI,EAAE;QACT,MAAM,GAAG,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAA;QACtC,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QAC9C,MAAM,OAAO,CAAC,WAAW,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;QACzC,+BAA+B;QAC/B,OAAO,CAAC,KAAK,EAAE,CAAA;QACf,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,QAAQ,eAAe,MAAM,CAAC,CAAA;QACrF,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAA;IACzC,CAAC,CACF,CAAA;IAED,EAAE,CAAC,iDAAiD,EAAE,KAAK,IAAI,EAAE;QAC/D,MAAM,GAAG,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAA;QACtC,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;QAC9C,MAAM,OAAO,CAAC,WAAW,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;QACzC,OAAO,CAAC,KAAK,EAAE,CAAA;QAEf,MAAM,QAAQ,GAAG,IAAI,CAAC,OAAO,EAAE,aAAa,EAAE,OAAO,EAAE,QAAQ,eAAe,MAAM,CAAC,CAAA;QACrF,IAAI,UAAU,CAAC,QAAQ,CAAC,EAAE,CAAC;YACzB,UAAU,CAAC,QAAQ,CAAC,CAAA;QACtB,CAAC;QAED,2DAA2D;QAC3D,MAAM,QAAQ,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;QAC7E,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,WAAW,CAAC,UAAU,EAAE,EAAE,CAAC,CAAA;QACzD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,CAAA;QACxC,QAAQ,CAAC,KAAK,EAAE,CAAA;IAClB,CAAC,CAAC,CAAA;IAEF,EAAE,CAAC,2DAA2D,EAAE,KAAK,IAAI,EAAE;QACzE,OAAO,CAAC,GAAG,CAAC,mBAAmB,GAAG,OAAO,CAAA;QACzC,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAA;YACtC,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;YAC9C,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,WAAW,CAAC,UAAU,EAAE,CAAC,CAAC,CAAA;YACvD,MAAM,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;QAC/B,CAAC;gBAAS,CAAC;YACT,OAAO,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAA;QACxC,CAAC;IACH,CAAC,CAAC,CAAA;AACJ,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SMI-4577: HNSW vs. brute-force microbench.
|
|
3
|
+
*
|
|
4
|
+
* Reports `bench()` timings for both backends. **Does NOT fail CI on
|
|
5
|
+
* regression** — vitest's bench mode is reporting-only. The hard CI gate
|
|
6
|
+
* lives in `hnsw-bench-gate.test.ts` which mirrors the same workload inside
|
|
7
|
+
* a `test()` and asserts the 5x p99 + recall@10 + rss-delta thresholds.
|
|
8
|
+
*
|
|
9
|
+
* Memory tracked via `process.memoryUsage().rss` (NOT `heapUsed` — hnswlib's
|
|
10
|
+
* graph lives in C++ memory and would undercount).
|
|
11
|
+
*
|
|
12
|
+
* Run: `docker exec skillsmith-dev-1 npm run bench:hnsw --workspace=@skillsmith/core`
|
|
13
|
+
*/
|
|
14
|
+
export {};
|
|
15
|
+
//# sourceMappingURL=hnsw-vs-brute-force.bench.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hnsw-vs-brute-force.bench.d.ts","sourceRoot":"","sources":["../../../tests/embeddings/hnsw-vs-brute-force.bench.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG"}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SMI-4577: HNSW vs. brute-force microbench.
|
|
3
|
+
*
|
|
4
|
+
* Reports `bench()` timings for both backends. **Does NOT fail CI on
|
|
5
|
+
* regression** — vitest's bench mode is reporting-only. The hard CI gate
|
|
6
|
+
* lives in `hnsw-bench-gate.test.ts` which mirrors the same workload inside
|
|
7
|
+
* a `test()` and asserts the 5x p99 + recall@10 + rss-delta thresholds.
|
|
8
|
+
*
|
|
9
|
+
* Memory tracked via `process.memoryUsage().rss` (NOT `heapUsed` — hnswlib's
|
|
10
|
+
* graph lives in C++ memory and would undercount).
|
|
11
|
+
*
|
|
12
|
+
* Run: `docker exec skillsmith-dev-1 npm run bench:hnsw --workspace=@skillsmith/core`
|
|
13
|
+
*/
|
|
14
|
+
import { describe, bench, beforeAll, afterAll } from 'vitest';
|
|
15
|
+
import { existsSync } from 'node:fs';
|
|
16
|
+
import { dirname, resolve } from 'node:path';
|
|
17
|
+
import { fileURLToPath } from 'node:url';
|
|
18
|
+
import { spawnSync } from 'node:child_process';
|
|
19
|
+
import { EmbeddingService } from '../../src/embeddings/index.js';
|
|
20
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
21
|
+
const FIXTURE_PATH = resolve(__dirname, 'fixtures', '14k-bench.db');
|
|
22
|
+
// Forced to mock embeddings throughout — the bench exercises the search
|
|
23
|
+
// backend, not the model.
|
|
24
|
+
process.env.SKILLSMITH_USE_MOCK_EMBEDDINGS = 'true';
|
|
25
|
+
let service;
|
|
26
|
+
const queryVectors = [];
|
|
27
|
+
async function ensureFixture() {
|
|
28
|
+
if (existsSync(FIXTURE_PATH))
|
|
29
|
+
return;
|
|
30
|
+
const seedScript = resolve(__dirname, 'seed-bench.ts');
|
|
31
|
+
const result = spawnSync('npx', ['tsx', seedScript], {
|
|
32
|
+
stdio: 'inherit',
|
|
33
|
+
cwd: resolve(__dirname, '..', '..'),
|
|
34
|
+
});
|
|
35
|
+
if (result.status !== 0) {
|
|
36
|
+
throw new Error(`seed-bench failed with exit code ${result.status}`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
beforeAll(async () => {
|
|
40
|
+
await ensureFixture();
|
|
41
|
+
service = await EmbeddingService.create({ dbPath: FIXTURE_PATH, useFallback: true });
|
|
42
|
+
// 50 random queries — enough samples for stable p99 without dominating
|
|
43
|
+
// bench warmup. Pulled from the same vector space so neighbourhoods exist.
|
|
44
|
+
const allEmbeddings = service.getAllEmbeddings();
|
|
45
|
+
const embeddings = Array.from(allEmbeddings.values());
|
|
46
|
+
for (let i = 0; i < 50; i++) {
|
|
47
|
+
queryVectors.push(embeddings[(i * 271) % embeddings.length]);
|
|
48
|
+
}
|
|
49
|
+
// Warm up the HNSW index so the bench measures steady-state search, not
|
|
50
|
+
// first-call build cost.
|
|
51
|
+
await service.findSimilar(queryVectors[0], 10);
|
|
52
|
+
});
|
|
53
|
+
afterAll(() => {
|
|
54
|
+
service?.close();
|
|
55
|
+
});
|
|
56
|
+
describe('findSimilar @ 14k vectors', () => {
|
|
57
|
+
bench('brute-force findSimilar topK=10', () => {
|
|
58
|
+
service.findSimilarBruteForce(queryVectors[Math.floor(Math.random() * queryVectors.length)], 10);
|
|
59
|
+
}, { iterations: 100, warmupIterations: 10 });
|
|
60
|
+
bench('hnsw findSimilar topK=10', async () => {
|
|
61
|
+
await service.findSimilar(queryVectors[Math.floor(Math.random() * queryVectors.length)], 10);
|
|
62
|
+
}, { iterations: 100, warmupIterations: 10 });
|
|
63
|
+
});
|
|
64
|
+
//# sourceMappingURL=hnsw-vs-brute-force.bench.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hnsw-vs-brute-force.bench.js","sourceRoot":"","sources":["../../../tests/embeddings/hnsw-vs-brute-force.bench.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,SAAS,EAAE,QAAQ,EAAE,MAAM,QAAQ,CAAA;AAC7D,OAAO,EAAE,UAAU,EAAE,MAAM,SAAS,CAAA;AACpC,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,SAAS,EAAE,MAAM,oBAAoB,CAAA;AAC9C,OAAO,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAA;AAEhE,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AACzD,MAAM,YAAY,GAAG,OAAO,CAAC,SAAS,EAAE,UAAU,EAAE,cAAc,CAAC,CAAA;AAEnE,wEAAwE;AACxE,0BAA0B;AAC1B,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,MAAM,CAAA;AAEnD,IAAI,OAAyB,CAAA;AAC7B,MAAM,YAAY,GAAmB,EAAE,CAAA;AAEvC,KAAK,UAAU,aAAa;IAC1B,IAAI,UAAU,CAAC,YAAY,CAAC;QAAE,OAAM;IACpC,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,EAAE,eAAe,CAAC,CAAA;IACtD,MAAM,MAAM,GAAG,SAAS,CAAC,KAAK,EAAE,CAAC,KAAK,EAAE,UAAU,CAAC,EAAE;QACnD,KAAK,EAAE,SAAS;QAChB,GAAG,EAAE,OAAO,CAAC,SAAS,EAAE,IAAI,EAAE,IAAI,CAAC;KACpC,CAAC,CAAA;IACF,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,MAAM,IAAI,KAAK,CAAC,oCAAoC,MAAM,CAAC,MAAM,EAAE,CAAC,CAAA;IACtE,CAAC;AACH,CAAC;AAED,SAAS,CAAC,KAAK,IAAI,EAAE;IACnB,MAAM,aAAa,EAAE,CAAA;IACrB,OAAO,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;IAEpF,uEAAuE;IACvE,2EAA2E;IAC3E,MAAM,aAAa,GAAG,OAAO,CAAC,gBAAgB,EAAE,CAAA;IAChD,MAAM,UAAU,GAAG,KAAK,CAAC,IAAI,CAAC,aAAa,CAAC,MAAM,EAAE,CAAC,CAAA;IACrD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,EAAE,EAAE,CAAC,EAAE,EAAE,CAAC;QAC5B,YAAY,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC,CAAA;IAC9D,CAAC;IAED,wEAAwE;IACxE,yBAAyB;IACzB,MAAM,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;AAChD,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,GAAG,EAAE;IACZ,OAAO,EAAE,KAAK,EAAE,CAAA;AAClB,CAAC,CAAC,CAAA;AAEF,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,KAAK,CACH,iCAAiC,EACjC,GAAG,EAAE;QACH,OAAO,CAAC,qBAAqB,CAC3B,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,EAC7D,EAAE,CACH,CAAA;IACH,CAAC,EACD,EAAE,UAAU,EAAE,GAAG,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAC1C,CAAA;IAED,KAAK,CACH,0BAA0B,EAC1B,KAAK,IAAI,EAAE;QACT,MAAM,OAAO,CAAC,WAAW,CAAC,YAAY,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,MAAM,EAAE,GAAG,YAAY,CAAC,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,CAAA;IAC9F,CAAC,EACD,EAAE,UAAU,EAAE,GAAG,EAAE,gBAAgB,EAAE,EAAE,EAAE,CAC1C,CAAA;AACH,CAAC,CAAC,CAAA"}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SMI-4577: Seed a synthetic 14k-skill embedding fixture for the HNSW bench.
|
|
3
|
+
*
|
|
4
|
+
* Generates deterministic mock embeddings via `EmbeddingService` (with
|
|
5
|
+
* `SKILLSMITH_USE_MOCK_EMBEDDINGS=true` for speed) and writes them to a
|
|
6
|
+
* SQLite cache that the bench reads back without network/model load.
|
|
7
|
+
*
|
|
8
|
+
* Run via: `npm run bench:hnsw:seed --workspace=@skillsmith/core`
|
|
9
|
+
*
|
|
10
|
+
* The fixture is gitignored (see top-level `.gitignore`) so each environment
|
|
11
|
+
* regenerates it on first bench run. CI invokes this as a `pretest` hook so
|
|
12
|
+
* the bench can boot from a clean checkout.
|
|
13
|
+
*/
|
|
14
|
+
export {};
|
|
15
|
+
//# sourceMappingURL=seed-bench.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"seed-bench.d.ts","sourceRoot":"","sources":["../../../tests/embeddings/seed-bench.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* SMI-4577: Seed a synthetic 14k-skill embedding fixture for the HNSW bench.
|
|
3
|
+
*
|
|
4
|
+
* Generates deterministic mock embeddings via `EmbeddingService` (with
|
|
5
|
+
* `SKILLSMITH_USE_MOCK_EMBEDDINGS=true` for speed) and writes them to a
|
|
6
|
+
* SQLite cache that the bench reads back without network/model load.
|
|
7
|
+
*
|
|
8
|
+
* Run via: `npm run bench:hnsw:seed --workspace=@skillsmith/core`
|
|
9
|
+
*
|
|
10
|
+
* The fixture is gitignored (see top-level `.gitignore`) so each environment
|
|
11
|
+
* regenerates it on first bench run. CI invokes this as a `pretest` hook so
|
|
12
|
+
* the bench can boot from a clean checkout.
|
|
13
|
+
*/
|
|
14
|
+
import { existsSync, mkdirSync, rmSync } from 'node:fs';
|
|
15
|
+
import { dirname, resolve } from 'node:path';
|
|
16
|
+
import { fileURLToPath } from 'node:url';
|
|
17
|
+
import { EmbeddingService } from '../../src/embeddings/index.js';
|
|
18
|
+
const __dirname = dirname(fileURLToPath(import.meta.url));
|
|
19
|
+
const FIXTURE_PATH = resolve(__dirname, 'fixtures', '14k-bench.db');
|
|
20
|
+
const TARGET_COUNT = 14_000;
|
|
21
|
+
async function main() {
|
|
22
|
+
// Force mock-embedding mode so the seed runs in <1s instead of pulling
|
|
23
|
+
// the all-MiniLM-L6-v2 model. The bench reads vectors back as raw
|
|
24
|
+
// Float32Array — semantic correctness is checked separately in the
|
|
25
|
+
// integration test.
|
|
26
|
+
process.env.SKILLSMITH_USE_MOCK_EMBEDDINGS = 'true';
|
|
27
|
+
mkdirSync(dirname(FIXTURE_PATH), { recursive: true });
|
|
28
|
+
if (existsSync(FIXTURE_PATH)) {
|
|
29
|
+
rmSync(FIXTURE_PATH);
|
|
30
|
+
}
|
|
31
|
+
const service = await EmbeddingService.create({ dbPath: FIXTURE_PATH, useFallback: true });
|
|
32
|
+
// Generate 14k synthetic skills. We oversample 100 base templates with
|
|
33
|
+
// numeric suffixes so vectors form natural clusters (a few "neighbours"
|
|
34
|
+
// per query) rather than uniform random — gives the bench a realistic
|
|
35
|
+
// recall workload.
|
|
36
|
+
const templates = Array.from({ length: 100 }, (_, i) => ({
|
|
37
|
+
id: `template-${i}`,
|
|
38
|
+
text: `Skill template ${i}: testing automation framework category ${i % 12} ${i % 7}`,
|
|
39
|
+
}));
|
|
40
|
+
const skills = [];
|
|
41
|
+
for (let i = 0; i < TARGET_COUNT; i++) {
|
|
42
|
+
const t = templates[i % templates.length];
|
|
43
|
+
skills.push({
|
|
44
|
+
id: `${t.id}-variant-${Math.floor(i / templates.length)}`,
|
|
45
|
+
text: `${t.text} variant ${i}`,
|
|
46
|
+
});
|
|
47
|
+
}
|
|
48
|
+
const start = Date.now();
|
|
49
|
+
const batch = await service.embedBatch(skills.map(({ id, text }) => ({ id, text })));
|
|
50
|
+
for (const { skillId, embedding, text } of batch) {
|
|
51
|
+
service.storeEmbedding(skillId, embedding, text);
|
|
52
|
+
}
|
|
53
|
+
const elapsedMs = Date.now() - start;
|
|
54
|
+
service.close();
|
|
55
|
+
console.log(`[seed-bench] wrote ${batch.length} embeddings to ${FIXTURE_PATH} in ${elapsedMs}ms`);
|
|
56
|
+
}
|
|
57
|
+
main().catch((err) => {
|
|
58
|
+
console.error('[seed-bench] failed:', err);
|
|
59
|
+
process.exit(1);
|
|
60
|
+
});
|
|
61
|
+
//# sourceMappingURL=seed-bench.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"seed-bench.js","sourceRoot":"","sources":["../../../tests/embeddings/seed-bench.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;GAYG;AAEH,OAAO,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,SAAS,CAAA;AACvD,OAAO,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAA;AAC5C,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAA;AACxC,OAAO,EAAE,gBAAgB,EAAE,MAAM,+BAA+B,CAAA;AAEhE,MAAM,SAAS,GAAG,OAAO,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAA;AACzD,MAAM,YAAY,GAAG,OAAO,CAAC,SAAS,EAAE,UAAU,EAAE,cAAc,CAAC,CAAA;AACnE,MAAM,YAAY,GAAG,MAAM,CAAA;AAE3B,KAAK,UAAU,IAAI;IACjB,uEAAuE;IACvE,kEAAkE;IAClE,mEAAmE;IACnE,oBAAoB;IACpB,OAAO,CAAC,GAAG,CAAC,8BAA8B,GAAG,MAAM,CAAA;IAEnD,SAAS,CAAC,OAAO,CAAC,YAAY,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IACrD,IAAI,UAAU,CAAC,YAAY,CAAC,EAAE,CAAC;QAC7B,MAAM,CAAC,YAAY,CAAC,CAAA;IACtB,CAAC;IAED,MAAM,OAAO,GAAG,MAAM,gBAAgB,CAAC,MAAM,CAAC,EAAE,MAAM,EAAE,YAAY,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC,CAAA;IAE1F,uEAAuE;IACvE,wEAAwE;IACxE,sEAAsE;IACtE,mBAAmB;IACnB,MAAM,SAAS,GAAG,KAAK,CAAC,IAAI,CAAC,EAAE,MAAM,EAAE,GAAG,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QACvD,EAAE,EAAE,YAAY,CAAC,EAAE;QACnB,IAAI,EAAE,kBAAkB,CAAC,2CAA2C,CAAC,GAAG,EAAE,IAAI,CAAC,GAAG,CAAC,EAAE;KACtF,CAAC,CAAC,CAAA;IAEH,MAAM,MAAM,GAAwC,EAAE,CAAA;IACtD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,YAAY,EAAE,CAAC,EAAE,EAAE,CAAC;QACtC,MAAM,CAAC,GAAG,SAAS,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,CAAA;QACzC,MAAM,CAAC,IAAI,CAAC;YACV,EAAE,EAAE,GAAG,CAAC,CAAC,EAAE,YAAY,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,SAAS,CAAC,MAAM,CAAC,EAAE;YACzD,IAAI,EAAE,GAAG,CAAC,CAAC,IAAI,YAAY,CAAC,EAAE;SAC/B,CAAC,CAAA;IACJ,CAAC;IAED,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAA;IACxB,MAAM,KAAK,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC,EAAE,EAAE,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,CAAA;IACpF,KAAK,MAAM,EAAE,OAAO,EAAE,SAAS,EAAE,IAAI,EAAE,IAAI,KAAK,EAAE,CAAC;QACjD,OAAO,CAAC,cAAc,CAAC,OAAO,EAAE,SAAS,EAAE,IAAI,CAAC,CAAA;IAClD,CAAC;IACD,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAA;IAEpC,OAAO,CAAC,KAAK,EAAE,CAAA;IAEf,OAAO,CAAC,GAAG,CAAC,sBAAsB,KAAK,CAAC,MAAM,kBAAkB,YAAY,OAAO,SAAS,IAAI,CAAC,CAAA;AACnG,CAAC;AAED,IAAI,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;IACnB,OAAO,CAAC,KAAK,CAAC,sBAAsB,EAAE,GAAG,CAAC,CAAA;IAC1C,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAA;AACjB,CAAC,CAAC,CAAA"}
|