mrplex 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +284 -0
- package/dist/cli/config.d.ts +28 -0
- package/dist/cli/config.js +48 -0
- package/dist/cli/config.js.map +1 -0
- package/dist/cli/exit-codes.d.ts +13 -0
- package/dist/cli/exit-codes.js +39 -0
- package/dist/cli/exit-codes.js.map +1 -0
- package/dist/cli/format.d.ts +22 -0
- package/dist/cli/format.js +148 -0
- package/dist/cli/format.js.map +1 -0
- package/dist/cli/main.d.ts +16 -0
- package/dist/cli/main.js +1518 -0
- package/dist/cli/main.js.map +1 -0
- package/dist/client/kernel-client.d.ts +97 -0
- package/dist/client/kernel-client.js +14 -0
- package/dist/client/kernel-client.js.map +1 -0
- package/dist/client/local.d.ts +26 -0
- package/dist/client/local.js +103 -0
- package/dist/client/local.js.map +1 -0
- package/dist/client/remote-mcp.d.ts +35 -0
- package/dist/client/remote-mcp.js +174 -0
- package/dist/client/remote-mcp.js.map +1 -0
- package/dist/embed/backfill.d.ts +24 -0
- package/dist/embed/backfill.js +33 -0
- package/dist/embed/backfill.js.map +1 -0
- package/dist/embed/chunker.d.ts +31 -0
- package/dist/embed/chunker.js +121 -0
- package/dist/embed/chunker.js.map +1 -0
- package/dist/embed/cmd-hook.d.ts +21 -0
- package/dist/embed/cmd-hook.js +100 -0
- package/dist/embed/cmd-hook.js.map +1 -0
- package/dist/embed/config.d.ts +61 -0
- package/dist/embed/config.js +109 -0
- package/dist/embed/config.js.map +1 -0
- package/dist/embed/hook.d.ts +38 -0
- package/dist/embed/hook.js +56 -0
- package/dist/embed/hook.js.map +1 -0
- package/dist/embed/http-hook.d.ts +13 -0
- package/dist/embed/http-hook.js +39 -0
- package/dist/embed/http-hook.js.map +1 -0
- package/dist/embed/worker.d.ts +62 -0
- package/dist/embed/worker.js +228 -0
- package/dist/embed/worker.js.map +1 -0
- package/dist/fixtures/hwe/characters/albion-ever.md +12 -0
- package/dist/fixtures/hwe/characters/cent.md +11 -0
- package/dist/fixtures/hwe/concepts/cent-demiurge-resonance.md +10 -0
- package/dist/fixtures/hwe/concepts/phase-looper.md +10 -0
- package/dist/fixtures/hwe/entities/demiurge.md +11 -0
- package/dist/fixtures/hwe/entities/monad.md +11 -0
- package/dist/fixtures/hwe/moc/globocorp-network.md +19 -0
- package/dist/fixtures/hwe/organizations/globocorp.md +11 -0
- package/dist/fixtures/hwe/readme.md +24 -0
- package/dist/fixtures/starship/crew/aria-okonkwo.md +14 -0
- package/dist/fixtures/starship/crew/bexley-orr.md +14 -0
- package/dist/fixtures/starship/crew/dax-thorne.md +14 -0
- package/dist/fixtures/starship/crew/isolde-marsh.md +16 -0
- package/dist/fixtures/starship/crew/kestrel-vance.md +15 -0
- package/dist/fixtures/starship/crew/quill-vasquez.md +14 -0
- package/dist/fixtures/starship/crew/soren-halloway.md +16 -0
- package/dist/fixtures/starship/encounters/the-drift-choir.md +14 -0
- package/dist/fixtures/starship/encounters/the-hollow-signal.md +16 -0
- package/dist/fixtures/starship/encounters/the-wandering-buoy.md +14 -0
- package/dist/fixtures/starship/equipment/coolant-loop-b.md +13 -0
- package/dist/fixtures/starship/equipment/emergency-ration-bars.md +12 -0
- package/dist/fixtures/starship/equipment/long-range-sensors.md +13 -0
- package/dist/fixtures/starship/equipment/plasma-manifold-3.md +15 -0
- package/dist/fixtures/starship/equipment/shuttle-corvid.md +13 -0
- package/dist/fixtures/starship/logs/marsh-4419-1.md +13 -0
- package/dist/fixtures/starship/logs/okonkwo-4415-1.md +13 -0
- package/dist/fixtures/starship/logs/orr-4413-1.md +13 -0
- package/dist/fixtures/starship/logs/orr-4420-1.md +12 -0
- package/dist/fixtures/starship/logs/thorne-4413-2.md +13 -0
- package/dist/fixtures/starship/logs/vance-4419-2.md +13 -0
- package/dist/fixtures/starship/misc/the-galley-ficus.md +11 -0
- package/dist/fixtures/starship/missions/the-cinder-run.md +17 -0
- package/dist/fixtures/starship/missions/the-drift-choir.md +17 -0
- package/dist/fixtures/starship/missions/the-hollow-signal.md +17 -0
- package/dist/fixtures/starship/missions/the-silent-beacon.md +17 -0
- package/dist/fixtures/starship/moc/crew.md +19 -0
- package/dist/fixtures/starship/moc/encounters.md +14 -0
- package/dist/fixtures/starship/moc/missions.md +15 -0
- package/dist/fixtures/starship/readme.md +60 -0
- package/dist/index.d.ts +1 -0
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -0
- package/dist/kernel/auth/glob.d.ts +46 -0
- package/dist/kernel/auth/glob.js +112 -0
- package/dist/kernel/auth/glob.js.map +1 -0
- package/dist/kernel/auth/scope.d.ts +53 -0
- package/dist/kernel/auth/scope.js +83 -0
- package/dist/kernel/auth/scope.js.map +1 -0
- package/dist/kernel/casefold.d.ts +37 -0
- package/dist/kernel/casefold.js +44 -0
- package/dist/kernel/casefold.js.map +1 -0
- package/dist/kernel/constants.d.ts +21 -0
- package/dist/kernel/constants.js +22 -0
- package/dist/kernel/constants.js.map +1 -0
- package/dist/kernel/context.d.ts +70 -0
- package/dist/kernel/context.js +114 -0
- package/dist/kernel/context.js.map +1 -0
- package/dist/kernel/deletion.d.ts +50 -0
- package/dist/kernel/deletion.js +83 -0
- package/dist/kernel/deletion.js.map +1 -0
- package/dist/kernel/diff.d.ts +36 -0
- package/dist/kernel/diff.js +61 -0
- package/dist/kernel/diff.js.map +1 -0
- package/dist/kernel/errors.d.ts +37 -0
- package/dist/kernel/errors.js +55 -0
- package/dist/kernel/errors.js.map +1 -0
- package/dist/kernel/frontmatter-input.d.ts +32 -0
- package/dist/kernel/frontmatter-input.js +85 -0
- package/dist/kernel/frontmatter-input.js.map +1 -0
- package/dist/kernel/graph.d.ts +34 -0
- package/dist/kernel/graph.js +559 -0
- package/dist/kernel/graph.js.map +1 -0
- package/dist/kernel/history.d.ts +74 -0
- package/dist/kernel/history.js +143 -0
- package/dist/kernel/history.js.map +1 -0
- package/dist/kernel/kernel.d.ts +162 -0
- package/dist/kernel/kernel.js +675 -0
- package/dist/kernel/kernel.js.map +1 -0
- package/dist/kernel/path-config.d.ts +101 -0
- package/dist/kernel/path-config.js +171 -0
- package/dist/kernel/path-config.js.map +1 -0
- package/dist/kernel/query/ast.d.ts +24 -0
- package/dist/kernel/query/ast.js +52 -0
- package/dist/kernel/query/ast.js.map +1 -0
- package/dist/kernel/query/cel-parse.d.ts +37 -0
- package/dist/kernel/query/cel-parse.js +88 -0
- package/dist/kernel/query/cel-parse.js.map +1 -0
- package/dist/kernel/query/degrees.d.ts +31 -0
- package/dist/kernel/query/degrees.js +135 -0
- package/dist/kernel/query/degrees.js.map +1 -0
- package/dist/kernel/query/graph-ast.d.ts +51 -0
- package/dist/kernel/query/graph-ast.js +139 -0
- package/dist/kernel/query/graph-ast.js.map +1 -0
- package/dist/kernel/query/query.d.ts +56 -0
- package/dist/kernel/query/query.js +276 -0
- package/dist/kernel/query/query.js.map +1 -0
- package/dist/kernel/validation.d.ts +64 -0
- package/dist/kernel/validation.js +183 -0
- package/dist/kernel/validation.js.map +1 -0
- package/dist/kernel/version-id.d.ts +10 -0
- package/dist/kernel/version-id.js +21 -0
- package/dist/kernel/version-id.js.map +1 -0
- package/dist/kernel/wire.d.ts +148 -0
- package/dist/kernel/wire.js +6 -0
- package/dist/kernel/wire.js.map +1 -0
- package/dist/links/backfill.d.ts +28 -0
- package/dist/links/backfill.js +33 -0
- package/dist/links/backfill.js.map +1 -0
- package/dist/links/extract.d.ts +29 -0
- package/dist/links/extract.js +233 -0
- package/dist/links/extract.js.map +1 -0
- package/dist/links/link-config.d.ts +70 -0
- package/dist/links/link-config.js +107 -0
- package/dist/links/link-config.js.map +1 -0
- package/dist/links/maintain.d.ts +37 -0
- package/dist/links/maintain.js +91 -0
- package/dist/links/maintain.js.map +1 -0
- package/dist/links/repair.d.ts +46 -0
- package/dist/links/repair.js +96 -0
- package/dist/links/repair.js.map +1 -0
- package/dist/links/resolve.d.ts +41 -0
- package/dist/links/resolve.js +121 -0
- package/dist/links/resolve.js.map +1 -0
- package/dist/links/stale.d.ts +32 -0
- package/dist/links/stale.js +73 -0
- package/dist/links/stale.js.map +1 -0
- package/dist/markdown/content-hash.d.ts +34 -0
- package/dist/markdown/content-hash.js +46 -0
- package/dist/markdown/content-hash.js.map +1 -0
- package/dist/markdown/frontmatter.d.ts +69 -0
- package/dist/markdown/frontmatter.js +147 -0
- package/dist/markdown/frontmatter.js.map +1 -0
- package/dist/markdown/hash-backfill.d.ts +22 -0
- package/dist/markdown/hash-backfill.js +43 -0
- package/dist/markdown/hash-backfill.js.map +1 -0
- package/dist/mcp/query-syntax.d.ts +20 -0
- package/dist/mcp/query-syntax.js +194 -0
- package/dist/mcp/query-syntax.js.map +1 -0
- package/dist/mcp/render.d.ts +31 -0
- package/dist/mcp/render.js +110 -0
- package/dist/mcp/render.js.map +1 -0
- package/dist/mcp/server.d.ts +60 -0
- package/dist/mcp/server.js +170 -0
- package/dist/mcp/server.js.map +1 -0
- package/dist/mcp/tools.d.ts +93 -0
- package/dist/mcp/tools.js +1218 -0
- package/dist/mcp/tools.js.map +1 -0
- package/dist/rest/conditional.d.ts +37 -0
- package/dist/rest/conditional.js +58 -0
- package/dist/rest/conditional.js.map +1 -0
- package/dist/rest/negotiate.d.ts +33 -0
- package/dist/rest/negotiate.js +54 -0
- package/dist/rest/negotiate.js.map +1 -0
- package/dist/rest/routes.d.ts +37 -0
- package/dist/rest/routes.js +744 -0
- package/dist/rest/routes.js.map +1 -0
- package/dist/seed/fixture-seed.d.ts +35 -0
- package/dist/seed/fixture-seed.js +75 -0
- package/dist/seed/fixture-seed.js.map +1 -0
- package/dist/seed/starship-config.d.ts +3 -0
- package/dist/seed/starship-config.js +5 -0
- package/dist/seed/starship-config.js.map +1 -0
- package/dist/server/headers.d.ts +36 -0
- package/dist/server/headers.js +30 -0
- package/dist/server/headers.js.map +1 -0
- package/dist/server/http-error.d.ts +53 -0
- package/dist/server/http-error.js +104 -0
- package/dist/server/http-error.js.map +1 -0
- package/dist/server/serve.d.ts +56 -0
- package/dist/server/serve.js +135 -0
- package/dist/server/serve.js.map +1 -0
- package/dist/shell/audit.d.ts +32 -0
- package/dist/shell/audit.js +28 -0
- package/dist/shell/audit.js.map +1 -0
- package/dist/shell/guard.d.ts +50 -0
- package/dist/shell/guard.js +176 -0
- package/dist/shell/guard.js.map +1 -0
- package/dist/shell/keys.d.ts +36 -0
- package/dist/shell/keys.js +64 -0
- package/dist/shell/keys.js.map +1 -0
- package/dist/shell/login.d.ts +56 -0
- package/dist/shell/login.js +130 -0
- package/dist/shell/login.js.map +1 -0
- package/dist/shell/oidc.d.ts +62 -0
- package/dist/shell/oidc.js +102 -0
- package/dist/shell/oidc.js.map +1 -0
- package/dist/shell/policy.d.ts +100 -0
- package/dist/shell/policy.js +232 -0
- package/dist/shell/policy.js.map +1 -0
- package/dist/shell/proxy-policy.d.ts +34 -0
- package/dist/shell/proxy-policy.js +120 -0
- package/dist/shell/proxy-policy.js.map +1 -0
- package/dist/shell/proxy.d.ts +47 -0
- package/dist/shell/proxy.js +253 -0
- package/dist/shell/proxy.js.map +1 -0
- package/dist/shell/serve.d.ts +86 -0
- package/dist/shell/serve.js +0 -0
- package/dist/shell/serve.js.map +1 -0
- package/dist/shell/stdio.d.ts +55 -0
- package/dist/shell/stdio.js +63 -0
- package/dist/shell/stdio.js.map +1 -0
- package/dist/storage/registry.d.ts +20 -0
- package/dist/storage/registry.js +38 -0
- package/dist/storage/registry.js.map +1 -0
- package/dist/storage/search-plan.d.ts +93 -0
- package/dist/storage/search-plan.js +15 -0
- package/dist/storage/search-plan.js.map +1 -0
- package/dist/storage/types.d.ts +448 -0
- package/dist/storage/types.js +14 -0
- package/dist/storage/types.js.map +1 -0
- package/dist/storage/versions-since.d.ts +72 -0
- package/dist/storage/versions-since.js +92 -0
- package/dist/storage/versions-since.js.map +1 -0
- package/dist/storage-postgres/adapter.d.ts +17 -0
- package/dist/storage-postgres/adapter.js +782 -0
- package/dist/storage-postgres/adapter.js.map +1 -0
- package/dist/storage-postgres/compile-postgres.d.ts +22 -0
- package/dist/storage-postgres/compile-postgres.js +830 -0
- package/dist/storage-postgres/compile-postgres.js.map +1 -0
- package/dist/storage-postgres/errors.d.ts +34 -0
- package/dist/storage-postgres/errors.js +50 -0
- package/dist/storage-postgres/errors.js.map +1 -0
- package/dist/storage-postgres/migrations/0001_init.sql +113 -0
- package/dist/storage-postgres/migrations/0002_content_hash.sql +7 -0
- package/dist/storage-postgres/migrations/index.d.ts +22 -0
- package/dist/storage-postgres/migrations/index.js +70 -0
- package/dist/storage-postgres/migrations/index.js.map +1 -0
- package/dist/storage-sqlite/adapter.d.ts +2 -0
- package/dist/storage-sqlite/adapter.js +688 -0
- package/dist/storage-sqlite/adapter.js.map +1 -0
- package/dist/storage-sqlite/compile-filter.d.ts +41 -0
- package/dist/storage-sqlite/compile-filter.js +881 -0
- package/dist/storage-sqlite/compile-filter.js.map +1 -0
- package/dist/storage-sqlite/compile-sqlite.d.ts +19 -0
- package/dist/storage-sqlite/compile-sqlite.js +176 -0
- package/dist/storage-sqlite/compile-sqlite.js.map +1 -0
- package/dist/storage-sqlite/migrations/0001_init.sql +123 -0
- package/dist/storage-sqlite/migrations/0002_content_hash.sql +7 -0
- package/dist/storage-sqlite/migrations/index.d.ts +6 -0
- package/dist/storage-sqlite/migrations/index.js +41 -0
- package/dist/storage-sqlite/migrations/index.js.map +1 -0
- package/dist/storage-sqlite/vec.d.ts +40 -0
- package/dist/storage-sqlite/vec.js +62 -0
- package/dist/storage-sqlite/vec.js.map +1 -0
- package/dist/sync/cursor.d.ts +36 -0
- package/dist/sync/cursor.js +66 -0
- package/dist/sync/cursor.js.map +1 -0
- package/dist/sync/daemon.d.ts +40 -0
- package/dist/sync/daemon.js +202 -0
- package/dist/sync/daemon.js.map +1 -0
- package/dist/sync/feed.d.ts +38 -0
- package/dist/sync/feed.js +125 -0
- package/dist/sync/feed.js.map +1 -0
- package/dist/sync/fs-store.d.ts +8 -0
- package/dist/sync/fs-store.js +76 -0
- package/dist/sync/fs-store.js.map +1 -0
- package/dist/sync/intrinsics.d.ts +52 -0
- package/dist/sync/intrinsics.js +76 -0
- package/dist/sync/intrinsics.js.map +1 -0
- package/dist/sync/paths.d.ts +38 -0
- package/dist/sync/paths.js +56 -0
- package/dist/sync/paths.js.map +1 -0
- package/dist/sync/push.d.ts +48 -0
- package/dist/sync/push.js +328 -0
- package/dist/sync/push.js.map +1 -0
- package/dist/sync/reconcile.d.ts +54 -0
- package/dist/sync/reconcile.js +252 -0
- package/dist/sync/reconcile.js.map +1 -0
- package/dist/sync/run.d.ts +27 -0
- package/dist/sync/run.js +46 -0
- package/dist/sync/run.js.map +1 -0
- package/dist/version.d.ts +2 -0
- package/dist/version.js +5 -0
- package/dist/version.js.map +1 -0
- package/package.json +80 -0
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Backfill: re-chunk + re-embed current versions missing chunks
|
|
3
|
+
* (design §5.3, m4-plan WS3).
|
|
4
|
+
*
|
|
5
|
+
* 1. Walk the repo's live versions.
|
|
6
|
+
* 2. For any without chunks (any model), enqueue.
|
|
7
|
+
* 3. Drain the backlog synchronously with a live worker.
|
|
8
|
+
*
|
|
9
|
+
* Deliberately does NOT delete stale chunks — the worker's
|
|
10
|
+
* chunks_upsert handles that on the re-embed. And deliberately does
|
|
11
|
+
* NOT re-embed versions that already have chunks — call `mrplex embed
|
|
12
|
+
* backfill --force` (future) to swap models. For v1, model changes go
|
|
13
|
+
* through: (a) point the hook at the new model, (b) delete the repo's
|
|
14
|
+
* chunks manually, (c) run backfill.
|
|
15
|
+
*/
|
|
16
|
+
export async function backfillRepo(storage, repoSlug, worker, onProgress) {
|
|
17
|
+
const repo = await storage.repos_by_slug(repoSlug);
|
|
18
|
+
if (!repo)
|
|
19
|
+
throw new Error(`repo not found: ${repoSlug}`);
|
|
20
|
+
const live = await storage.versions_live_by_repo(repo.id);
|
|
21
|
+
let enqueued = 0;
|
|
22
|
+
for (const version of live) {
|
|
23
|
+
const existing = await storage.chunks_by_version(version.id);
|
|
24
|
+
if (existing.length === 0) {
|
|
25
|
+
await storage.backlog_enqueue(version.id);
|
|
26
|
+
enqueued++;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
onProgress?.(`enqueued ${enqueued} version(s)`);
|
|
30
|
+
const { processed, failed, skipped } = await worker.drainOnce();
|
|
31
|
+
return { enqueued, processed, failed, skipped };
|
|
32
|
+
}
|
|
33
|
+
//# sourceMappingURL=backfill.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"backfill.js","sourceRoot":"","sources":["../../src/embed/backfill.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;GAcG;AAYH,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,OAAgB,EAChB,QAAgB,EAChB,MAAc,EACd,UAAkC;IAElC,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,aAAa,CAAC,QAAQ,CAAC,CAAC;IACnD,IAAI,CAAC,IAAI;QAAE,MAAM,IAAI,KAAK,CAAC,mBAAmB,QAAQ,EAAE,CAAC,CAAC;IAC1D,MAAM,IAAI,GAAG,MAAM,OAAO,CAAC,qBAAqB,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAC1D,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,KAAK,MAAM,OAAO,IAAI,IAAI,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,iBAAiB,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;QAC7D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,OAAO,CAAC,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC;YAC1C,QAAQ,EAAE,CAAC;QACb,CAAC;IACH,CAAC;IACD,UAAU,EAAE,CAAC,YAAY,QAAQ,aAAa,CAAC,CAAC;IAChD,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,MAAM,MAAM,CAAC,SAAS,EAAE,CAAC;IAChE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;AAClD,CAAC"}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic markdown-body chunker (design §5.3, m4-plan §5 decision 7).
|
|
3
|
+
*
|
|
4
|
+
* Contract:
|
|
5
|
+
* • Same input → same chunks. Always. Dedup by (model, text_hash)
|
|
6
|
+
* relies on this and only this — a change here re-embeds everything.
|
|
7
|
+
* • Body-only. Frontmatter is the structured/query side (§3.2); the
|
|
8
|
+
* hook never sees it.
|
|
9
|
+
* • Splits on ATX heading and blank-line block boundaries; greedy-packs
|
|
10
|
+
* consecutive blocks up to MAX_CHUNK_CHARS; a single block exceeding
|
|
11
|
+
* the cap hard-splits at the cap.
|
|
12
|
+
* • `text_hash` = sha256(chunk_text) — same primitive as token hashing.
|
|
13
|
+
*
|
|
14
|
+
* The chunk-size constant is code, not config: `text_hash` dedup keys
|
|
15
|
+
* would silently invalidate on a config change, and users of `rank`
|
|
16
|
+
* would see quiet drift in ranking behavior across servers. Revisiting
|
|
17
|
+
* chunk size later only costs one backfill (§5.3, `embed backfill`).
|
|
18
|
+
*/
|
|
19
|
+
/** Maximum characters per chunk. Constant, not config; see file header. */
|
|
20
|
+
export declare const MAX_CHUNK_CHARS = 2000;
|
|
21
|
+
export type Chunk = {
|
|
22
|
+
/** 0-based document order. Contiguous within a chunk[] result. */
|
|
23
|
+
ix: number;
|
|
24
|
+
text: string;
|
|
25
|
+
text_hash: string;
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* Chunk a document body. Empty / whitespace-only body → empty array
|
|
29
|
+
* (a valid state; the doc simply never ranks — see §5.3).
|
|
30
|
+
*/
|
|
31
|
+
export declare function chunkBody(body: string): Chunk[];
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Deterministic markdown-body chunker (design §5.3, m4-plan §5 decision 7).
|
|
3
|
+
*
|
|
4
|
+
* Contract:
|
|
5
|
+
* • Same input → same chunks. Always. Dedup by (model, text_hash)
|
|
6
|
+
* relies on this and only this — a change here re-embeds everything.
|
|
7
|
+
* • Body-only. Frontmatter is the structured/query side (§3.2); the
|
|
8
|
+
* hook never sees it.
|
|
9
|
+
* • Splits on ATX heading and blank-line block boundaries; greedy-packs
|
|
10
|
+
* consecutive blocks up to MAX_CHUNK_CHARS; a single block exceeding
|
|
11
|
+
* the cap hard-splits at the cap.
|
|
12
|
+
* • `text_hash` = sha256(chunk_text) — same primitive as token hashing.
|
|
13
|
+
*
|
|
14
|
+
* The chunk-size constant is code, not config: `text_hash` dedup keys
|
|
15
|
+
* would silently invalidate on a config change, and users of `rank`
|
|
16
|
+
* would see quiet drift in ranking behavior across servers. Revisiting
|
|
17
|
+
* chunk size later only costs one backfill (§5.3, `embed backfill`).
|
|
18
|
+
*/
|
|
19
|
+
import { createHash } from "node:crypto";
|
|
20
|
+
/** Maximum characters per chunk. Constant, not config; see file header. */
|
|
21
|
+
export const MAX_CHUNK_CHARS = 2000;
|
|
22
|
+
/**
|
|
23
|
+
* Chunk a document body. Empty / whitespace-only body → empty array
|
|
24
|
+
* (a valid state; the doc simply never ranks — see §5.3).
|
|
25
|
+
*/
|
|
26
|
+
export function chunkBody(body) {
|
|
27
|
+
const blocks = splitBlocks(body);
|
|
28
|
+
const packed = greedyPack(blocks, MAX_CHUNK_CHARS);
|
|
29
|
+
return packed.map((text, ix) => ({
|
|
30
|
+
ix,
|
|
31
|
+
text,
|
|
32
|
+
text_hash: sha256(text),
|
|
33
|
+
}));
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* Split into blocks at ATX heading and blank-line boundaries. Blank
|
|
37
|
+
* lines are ONLY the boundary — they don't join the output.
|
|
38
|
+
*
|
|
39
|
+
* "Block" here is coarse: consecutive non-blank lines belong to one
|
|
40
|
+
* block; an ATX heading line starts a fresh block. That covers the
|
|
41
|
+
* common markdown shapes (paragraphs, lists, headings, code fences)
|
|
42
|
+
* without a full commonmark parse. If a code fence contains an
|
|
43
|
+
* accidental `#`-leading line, the heading rule mis-splits — a chunker
|
|
44
|
+
* is not a renderer, and this is worse cosmetically than semantically:
|
|
45
|
+
* the split still deterministically produces the same chunks every
|
|
46
|
+
* time, which is what dedup requires.
|
|
47
|
+
*/
|
|
48
|
+
function splitBlocks(body) {
|
|
49
|
+
const lines = body.split(/\r?\n/);
|
|
50
|
+
const blocks = [];
|
|
51
|
+
let current = [];
|
|
52
|
+
const push = () => {
|
|
53
|
+
if (current.length === 0)
|
|
54
|
+
return;
|
|
55
|
+
const joined = current.join("\n").replace(/\s+$/, "");
|
|
56
|
+
if (joined.length > 0)
|
|
57
|
+
blocks.push(joined);
|
|
58
|
+
current = [];
|
|
59
|
+
};
|
|
60
|
+
for (const line of lines) {
|
|
61
|
+
const isBlank = line.trim().length === 0;
|
|
62
|
+
const isHeading = /^\s{0,3}#{1,6}\s/.test(line);
|
|
63
|
+
if (isBlank) {
|
|
64
|
+
push();
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
if (isHeading && current.length > 0) {
|
|
68
|
+
push();
|
|
69
|
+
}
|
|
70
|
+
current.push(line);
|
|
71
|
+
}
|
|
72
|
+
push();
|
|
73
|
+
return blocks;
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Greedy-pack blocks into chunks of at most `cap` characters. A single
|
|
77
|
+
* block over `cap` hard-splits at exactly `cap` byte-wise on characters
|
|
78
|
+
* (JS string units) — no attempt to preserve word boundaries. Reason:
|
|
79
|
+
* an accidental split is far preferable to a per-corpus non-deterministic
|
|
80
|
+
* split rule (dedup sensitivity, §5.3).
|
|
81
|
+
*/
|
|
82
|
+
function greedyPack(blocks, cap) {
|
|
83
|
+
const chunks = [];
|
|
84
|
+
let buf = "";
|
|
85
|
+
const flush = () => {
|
|
86
|
+
if (buf.length === 0)
|
|
87
|
+
return;
|
|
88
|
+
chunks.push(buf);
|
|
89
|
+
buf = "";
|
|
90
|
+
};
|
|
91
|
+
for (const block of blocks) {
|
|
92
|
+
if (block.length > cap) {
|
|
93
|
+
// Flush anything we were building; hard-split the oversized block.
|
|
94
|
+
flush();
|
|
95
|
+
for (let i = 0; i < block.length; i += cap) {
|
|
96
|
+
chunks.push(block.slice(i, i + cap));
|
|
97
|
+
}
|
|
98
|
+
continue;
|
|
99
|
+
}
|
|
100
|
+
if (buf.length === 0) {
|
|
101
|
+
buf = block;
|
|
102
|
+
continue;
|
|
103
|
+
}
|
|
104
|
+
// Join with a blank line so a two-block chunk still reads as
|
|
105
|
+
// two blocks — hooks that stem/tokenize benefit from the separator.
|
|
106
|
+
const candidate = `${buf}\n\n${block}`;
|
|
107
|
+
if (candidate.length <= cap) {
|
|
108
|
+
buf = candidate;
|
|
109
|
+
}
|
|
110
|
+
else {
|
|
111
|
+
flush();
|
|
112
|
+
buf = block;
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
flush();
|
|
116
|
+
return chunks;
|
|
117
|
+
}
|
|
118
|
+
function sha256(input) {
|
|
119
|
+
return createHash("sha256").update(input, "utf8").digest("hex");
|
|
120
|
+
}
|
|
121
|
+
//# sourceMappingURL=chunker.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"chunker.js","sourceRoot":"","sources":["../../src/embed/chunker.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;GAiBG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAEzC,2EAA2E;AAC3E,MAAM,CAAC,MAAM,eAAe,GAAG,IAAI,CAAC;AASpC;;;GAGG;AACH,MAAM,UAAU,SAAS,CAAC,IAAY;IACpC,MAAM,MAAM,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;IACjC,MAAM,MAAM,GAAG,UAAU,CAAC,MAAM,EAAE,eAAe,CAAC,CAAC;IACnD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,EAAE,EAAE,EAAE,CAAC,CAAC;QAC/B,EAAE;QACF,IAAI;QACJ,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC;KACxB,CAAC,CAAC,CAAC;AACN,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,SAAS,WAAW,CAAC,IAAY;IAC/B,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAClC,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,OAAO,GAAa,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAG,GAAG,EAAE;QAChB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QACjC,MAAM,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACtD,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC3C,OAAO,GAAG,EAAE,CAAC;IACf,CAAC,CAAC;IACF,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,MAAM,OAAO,GAAG,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,CAAC;QACzC,MAAM,SAAS,GAAG,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,IAAI,OAAO,EAAE,CAAC;YACZ,IAAI,EAAE,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,SAAS,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,EAAE,CAAC;QACT,CAAC;QACD,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrB,CAAC;IACD,IAAI,EAAE,CAAC;IACP,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;;GAMG;AACH,SAAS,UAAU,CAAC,MAAyB,EAAE,GAAW;IACxD,MAAM,MAAM,GAAa,EAAE,CAAC;IAC5B,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,MAAM,KAAK,GAAG,GAAG,EAAE;QACjB,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO;QAC7B,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACjB,GAAG,GAAG,EAAE,CAAC;IACX,CAAC,CAAC;IACF,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,IAAI,KAAK,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;YACvB,mEAAmE;YACnE,KAAK,EAAE,CAAC;YACR,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,GAAG,EAAE,CAAC;gBAC3C,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC;YACvC,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,GAAG,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YACrB,GAAG,GAAG,KAAK,CAAC;YACZ,SAAS;QACX,CAAC;QACD,6DAA6D;QAC7D,oEAAoE;QACpE,MAAM,SAAS,GAAG,GAAG,GAAG,OAAO,KAAK,EAAE,CAAC;QACvC,IAAI,SAAS,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;YAC5B,GAAG,GAAG,SAAS,CAAC;QAClB,CAAC;aAAM,CAAC;YACN,KAAK,EAAE,CAAC;YACR,GAAG,GAAG,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IACD,KAAK,EAAE,CAAC;IACR,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,MAAM,CAAC,KAAa;IAC3B,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAClE,CAAC"}
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Subprocess embedding hook (--embed-cmd). Spawns a long-lived process
|
|
3
|
+
* once; writes one JSON line per batch to stdin; reads one JSON line
|
|
4
|
+
* back from stdout (design §5.3, m4-plan §5 decision 1).
|
|
5
|
+
*
|
|
6
|
+
* Stderr passes through to the server log; the hook doesn't parse it.
|
|
7
|
+
*
|
|
8
|
+
* Concurrency: at most one in-flight embed() call per hook instance.
|
|
9
|
+
* The worker's drain loop is single-flight per batch, so this is fine
|
|
10
|
+
* for v1 — a hook that needs parallelism can implement it internally.
|
|
11
|
+
*/
|
|
12
|
+
import { type EmbedHook } from "./hook.js";
|
|
13
|
+
export type CmdHookOptions = {
|
|
14
|
+
command: string;
|
|
15
|
+
/** Optional args if the command needs them (rare; usually a single script). */
|
|
16
|
+
args?: readonly string[];
|
|
17
|
+
timeoutMs?: number;
|
|
18
|
+
/** stderr sink — defaults to the parent's stderr. */
|
|
19
|
+
onStderr?: (chunk: string) => void;
|
|
20
|
+
};
|
|
21
|
+
export declare function createCmdEmbedHook(opts: CmdHookOptions): EmbedHook;
|
|
@@ -0,0 +1,100 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Subprocess embedding hook (--embed-cmd). Spawns a long-lived process
|
|
3
|
+
* once; writes one JSON line per batch to stdin; reads one JSON line
|
|
4
|
+
* back from stdout (design §5.3, m4-plan §5 decision 1).
|
|
5
|
+
*
|
|
6
|
+
* Stderr passes through to the server log; the hook doesn't parse it.
|
|
7
|
+
*
|
|
8
|
+
* Concurrency: at most one in-flight embed() call per hook instance.
|
|
9
|
+
* The worker's drain loop is single-flight per batch, so this is fine
|
|
10
|
+
* for v1 — a hook that needs parallelism can implement it internally.
|
|
11
|
+
*/
|
|
12
|
+
import { spawn } from "node:child_process";
|
|
13
|
+
import { validateEmbedResponse } from "./hook.js";
|
|
14
|
+
export function createCmdEmbedHook(opts) {
|
|
15
|
+
const timeout = opts.timeoutMs ?? 60_000;
|
|
16
|
+
const onStderr = opts.onStderr ?? ((c) => process.stderr.write(c));
|
|
17
|
+
let proc = null;
|
|
18
|
+
let readBuffer = "";
|
|
19
|
+
let pending = null;
|
|
20
|
+
let pendingError = null;
|
|
21
|
+
let inFlight = Promise.resolve();
|
|
22
|
+
function ensureProc() {
|
|
23
|
+
if (proc && proc.exitCode === null)
|
|
24
|
+
return proc;
|
|
25
|
+
proc = spawn(opts.command, opts.args ?? [], {
|
|
26
|
+
stdio: ["pipe", "pipe", "pipe"],
|
|
27
|
+
});
|
|
28
|
+
proc.stdout.setEncoding("utf8");
|
|
29
|
+
proc.stderr.setEncoding("utf8");
|
|
30
|
+
proc.stdout.on("data", (chunk) => {
|
|
31
|
+
readBuffer += chunk;
|
|
32
|
+
let nl = readBuffer.indexOf("\n");
|
|
33
|
+
while (nl >= 0) {
|
|
34
|
+
const line = readBuffer.slice(0, nl);
|
|
35
|
+
readBuffer = readBuffer.slice(nl + 1);
|
|
36
|
+
if (pending) {
|
|
37
|
+
const cb = pending;
|
|
38
|
+
pending = null;
|
|
39
|
+
pendingError = null;
|
|
40
|
+
cb(line);
|
|
41
|
+
}
|
|
42
|
+
nl = readBuffer.indexOf("\n");
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
proc.stderr.on("data", (chunk) => onStderr(chunk));
|
|
46
|
+
proc.on("exit", (code, signal) => {
|
|
47
|
+
const err = new Error(`embed hook subprocess exited (code=${code} signal=${signal ?? ""})`);
|
|
48
|
+
if (pendingError) {
|
|
49
|
+
const cb = pendingError;
|
|
50
|
+
pending = null;
|
|
51
|
+
pendingError = null;
|
|
52
|
+
cb(err);
|
|
53
|
+
}
|
|
54
|
+
proc = null;
|
|
55
|
+
readBuffer = "";
|
|
56
|
+
});
|
|
57
|
+
return proc;
|
|
58
|
+
}
|
|
59
|
+
return {
|
|
60
|
+
label: [opts.command, ...(opts.args ?? [])].join(" "),
|
|
61
|
+
async embed(chunks) {
|
|
62
|
+
// Serialize batches — one in-flight at a time. Awaiting the
|
|
63
|
+
// previous call's promise chains us behind it.
|
|
64
|
+
const gate = inFlight.catch(() => undefined);
|
|
65
|
+
inFlight = gate.then(async () => {
|
|
66
|
+
const p = ensureProc();
|
|
67
|
+
const linePromise = new Promise((resolve, reject) => {
|
|
68
|
+
pending = resolve;
|
|
69
|
+
pendingError = reject;
|
|
70
|
+
});
|
|
71
|
+
const timer = setTimeout(() => {
|
|
72
|
+
if (pendingError) {
|
|
73
|
+
const cb = pendingError;
|
|
74
|
+
pending = null;
|
|
75
|
+
pendingError = null;
|
|
76
|
+
cb(new Error(`embed hook timed out after ${timeout}ms`));
|
|
77
|
+
}
|
|
78
|
+
}, timeout);
|
|
79
|
+
try {
|
|
80
|
+
p.stdin.write(`${JSON.stringify({ chunks })}\n`);
|
|
81
|
+
const line = await linePromise;
|
|
82
|
+
const body = JSON.parse(line);
|
|
83
|
+
return validateEmbedResponse(body, chunks);
|
|
84
|
+
}
|
|
85
|
+
finally {
|
|
86
|
+
clearTimeout(timer);
|
|
87
|
+
}
|
|
88
|
+
});
|
|
89
|
+
return inFlight;
|
|
90
|
+
},
|
|
91
|
+
async close() {
|
|
92
|
+
if (proc && proc.exitCode === null) {
|
|
93
|
+
proc.stdin.end();
|
|
94
|
+
proc.kill();
|
|
95
|
+
}
|
|
96
|
+
proc = null;
|
|
97
|
+
},
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
//# sourceMappingURL=cmd-hook.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"cmd-hook.js","sourceRoot":"","sources":["../../src/embed/cmd-hook.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;GAUG;AAEH,OAAO,EAAuC,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAChF,OAAO,EAAsC,qBAAqB,EAAE,MAAM,WAAW,CAAC;AAWtF,MAAM,UAAU,kBAAkB,CAAC,IAAoB;IACrD,MAAM,OAAO,GAAG,IAAI,CAAC,SAAS,IAAI,MAAM,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAEnE,IAAI,IAAI,GAA0C,IAAI,CAAC;IACvD,IAAI,UAAU,GAAG,EAAE,CAAC;IACpB,IAAI,OAAO,GAAoC,IAAI,CAAC;IACpD,IAAI,YAAY,GAAkC,IAAI,CAAC;IACvD,IAAI,QAAQ,GAAqB,OAAO,CAAC,OAAO,EAAE,CAAC;IAEnD,SAAS,UAAU;QACjB,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,KAAK,IAAI;YAAE,OAAO,IAAI,CAAC;QAChD,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC,OAAO,EAAE,IAAI,CAAC,IAAI,IAAI,EAAE,EAAE;YAC1C,KAAK,EAAE,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,CAAC;SAChC,CAAmC,CAAC;QACrC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE;YACvC,UAAU,IAAI,KAAK,CAAC;YACpB,IAAI,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAClC,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;gBACf,MAAM,IAAI,GAAG,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACrC,UAAU,GAAG,UAAU,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;gBACtC,IAAI,OAAO,EAAE,CAAC;oBACZ,MAAM,EAAE,GAAG,OAAO,CAAC;oBACnB,OAAO,GAAG,IAAI,CAAC;oBACf,YAAY,GAAG,IAAI,CAAC;oBACpB,EAAE,CAAC,IAAI,CAAC,CAAC;gBACX,CAAC;gBACD,EAAE,GAAG,UAAU,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;YAChC,CAAC;QACH,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,KAAa,EAAE,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC;QAC3D,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,CAAC,IAAI,EAAE,MAAM,EAAE,EAAE;YAC/B,MAAM,GAAG,GAAG,IAAI,KAAK,CAAC,sCAAsC,IAAI,WAAW,MAAM,IAAI,EAAE,GAAG,CAAC,CAAC;YAC5F,IAAI,YAAY,EAAE,CAAC;gBACjB,MAAM,EAAE,GAAG,YAAY,CAAC;gBACxB,OAAO,GAAG,IAAI,CAAC;gBACf,YAAY,GAAG,IAAI,CAAC;gBACpB,EAAE,CAAC,GAAG,CAAC,CAAC;YACV,CAAC;YACD,IAAI,GAAG,IAAI,CAAC;YACZ,UAAU,GAAG,EAAE,CAAC;QAClB,CAAC,CAAC,CAAC;QACH,OAAO,IAAI,CAAC;IACd,CAAC;IAED,OAAO;QACL,KAAK,EAAE,CAAC,IAAI,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC;QACrD,KAAK,CAAC,KAAK,CAAC,MAAyB;YACnC,4DAA4D;YAC5D,+CAA+C;YAC/C,MAAM,IAAI,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,SAAS,CAAC,CAAC;YAC7C,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,IAAI,EAAE;gBAC9B,MAAM,CAAC,GAAG,UAAU,EAAE,CAAC;gBACvB,MAAM,WAAW,GAAG,IAAI,OAAO,CAAS,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;oBAC1D,OAAO,GAAG,OAAO,CAAC;oBAClB,YAAY,GAAG,MAAM,CAAC;gBACxB,CAAC,CAAC,CAAC;gBACH,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE;oBAC5B,IAAI,YAAY,EAAE,CAAC;wBACjB,MAAM,EAAE,GAAG,YAAY,CAAC;wBACxB,OAAO,GAAG,IAAI,CAAC;wBACf,YAAY,GAAG,IAAI,CAAC;wBACpB,EAAE,CAAC,IAAI,KAAK,CAAC,8BAA8B,OAAO,IAAI,CAAC,CAAC,CAAC;oBAC3D,CAAC;gBACH,CAAC,EAAE,OAAO,CAAC,CAAC;gBACZ,IAAI,CAAC;oBACH,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,MAAM,EAAE,CAAC,IAAI,CAAC,CAAC;oBACjD,MAAM,IAAI,GAAG,MAAM,WAAW,CAAC;oBAC/B,MAAM,IAAI,GAAY,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;oBACvC,OAAO,qBAAqB,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;gBAC7C,CAAC;wBAAS,CAAC;oBACT,YAAY,CAAC,KAAK,CAAC,CAAC;gBACtB,CAAC;YACH,CAAC,CAAC,CAAC;YACH,OAAO,QAAkC,CAAC;QAC5C,CAAC;QACD,KAAK,CAAC,KAAK;YACT,IAAI,IAAI,IAAI,IAAI,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;gBACnC,IAAI,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC;gBACjB,IAAI,CAAC,IAAI,EAAE,CAAC;YACd,CAAC;YACD,IAAI,GAAG,IAAI,CAAC;QACd,CAAC;KACF,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embed hook configuration resolution — shared between `mrplex serve`
|
|
3
|
+
* and the local `mrplex embed` commands (m4-plan WS3).
|
|
4
|
+
*
|
|
5
|
+
* Precedence per source kind: CLI flag → env var → CLI config file → unset.
|
|
6
|
+
* Only one source kind may be configured at a time (`--embedder`, legacy
|
|
7
|
+
* `--embed-url`, or legacy `--embed-cmd` / their env/config equivalents).
|
|
8
|
+
*/
|
|
9
|
+
import type { EmbedHook } from "./hook.js";
|
|
10
|
+
export type EmbedFlagInputs = {
|
|
11
|
+
/** Unified provider: subprocess command, or http(s):// URL. */
|
|
12
|
+
embedder?: string;
|
|
13
|
+
/** @deprecated Use `embedder` with an http(s):// URL. */
|
|
14
|
+
embed_url?: string;
|
|
15
|
+
/** @deprecated Use `embedder` with a command. */
|
|
16
|
+
embed_cmd?: string;
|
|
17
|
+
};
|
|
18
|
+
export type EmbedConfig = {
|
|
19
|
+
kind: "http";
|
|
20
|
+
url: string;
|
|
21
|
+
} | {
|
|
22
|
+
kind: "cmd";
|
|
23
|
+
command: string;
|
|
24
|
+
} | {
|
|
25
|
+
kind: "none";
|
|
26
|
+
};
|
|
27
|
+
/**
|
|
28
|
+
* Parse a unified `--embedder` value. Only `http://` and `https://` select
|
|
29
|
+
* the HTTP hook; everything else is a subprocess command string.
|
|
30
|
+
*/
|
|
31
|
+
export declare function parseEmbedderSpec(value: string): EmbedConfig;
|
|
32
|
+
export declare function resolveEmbedConfig(inputs?: EmbedFlagInputs): EmbedConfig;
|
|
33
|
+
export declare function describeEmbedConfig(c: EmbedConfig): string;
|
|
34
|
+
/**
|
|
35
|
+
* Instantiate a hook from a resolved config. Returns null if the
|
|
36
|
+
* config is `none` — callers decide whether that is fatal.
|
|
37
|
+
*/
|
|
38
|
+
export declare function createHookFromConfig(c: EmbedConfig): EmbedHook | null;
|
|
39
|
+
/** Commander option bundle for commands that accept an embedding provider. */
|
|
40
|
+
export declare const EMBED_CLI_OPTIONS: {
|
|
41
|
+
readonly embedder: {
|
|
42
|
+
readonly flags: "--embedder <spec>";
|
|
43
|
+
readonly description: "embedding provider: command, or http(s):// URL";
|
|
44
|
+
};
|
|
45
|
+
readonly embedUrl: {
|
|
46
|
+
readonly flags: "--embed-url <url>";
|
|
47
|
+
readonly description: "(deprecated) use --embedder with an http(s):// URL";
|
|
48
|
+
};
|
|
49
|
+
readonly embedCmd: {
|
|
50
|
+
readonly flags: "--embed-cmd <cmd>";
|
|
51
|
+
readonly description: "(deprecated) use --embedder with a command";
|
|
52
|
+
};
|
|
53
|
+
};
|
|
54
|
+
export type EmbedCliOpts = {
|
|
55
|
+
embedder?: string;
|
|
56
|
+
embedUrl?: string;
|
|
57
|
+
embedCmd?: string;
|
|
58
|
+
};
|
|
59
|
+
export declare function embedFlagInputsFromCli(opts: EmbedCliOpts): EmbedFlagInputs;
|
|
60
|
+
import type { Command } from "commander";
|
|
61
|
+
export declare function addEmbedCliOptions(cmd: Command): Command;
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embed hook configuration resolution — shared between `mrplex serve`
|
|
3
|
+
* and the local `mrplex embed` commands (m4-plan WS3).
|
|
4
|
+
*
|
|
5
|
+
* Precedence per source kind: CLI flag → env var → CLI config file → unset.
|
|
6
|
+
* Only one source kind may be configured at a time (`--embedder`, legacy
|
|
7
|
+
* `--embed-url`, or legacy `--embed-cmd` / their env/config equivalents).
|
|
8
|
+
*/
|
|
9
|
+
import { loadConfig } from "../cli/config.js";
|
|
10
|
+
import { createCmdEmbedHook } from "./cmd-hook.js";
|
|
11
|
+
import { createHttpEmbedHook } from "./http-hook.js";
|
|
12
|
+
/**
|
|
13
|
+
* Parse a unified `--embedder` value. Only `http://` and `https://` select
|
|
14
|
+
* the HTTP hook; everything else is a subprocess command string.
|
|
15
|
+
*/
|
|
16
|
+
export function parseEmbedderSpec(value) {
|
|
17
|
+
const trimmed = value.trim();
|
|
18
|
+
if (trimmed.length === 0) {
|
|
19
|
+
throw new Error("--embedder is empty");
|
|
20
|
+
}
|
|
21
|
+
if (trimmed.startsWith("http://") || trimmed.startsWith("https://")) {
|
|
22
|
+
return { kind: "http", url: trimmed };
|
|
23
|
+
}
|
|
24
|
+
return { kind: "cmd", command: trimmed };
|
|
25
|
+
}
|
|
26
|
+
export function resolveEmbedConfig(inputs = {}) {
|
|
27
|
+
const cfg = loadConfig();
|
|
28
|
+
const embedder = inputs.embedder ?? process.env.MRPLEX_EMBEDDER ?? cfg.embedder;
|
|
29
|
+
const url = inputs.embed_url ?? process.env.MRPLEX_EMBED_URL ?? cfg.embed_url;
|
|
30
|
+
const cmd = inputs.embed_cmd ?? process.env.MRPLEX_EMBED_CMD ?? cfg.embed_cmd;
|
|
31
|
+
const sources = [];
|
|
32
|
+
if (embedder)
|
|
33
|
+
sources.push("--embedder");
|
|
34
|
+
if (url)
|
|
35
|
+
sources.push("--embed-url");
|
|
36
|
+
if (cmd)
|
|
37
|
+
sources.push("--embed-cmd");
|
|
38
|
+
if (sources.length > 1) {
|
|
39
|
+
throw new Error(`${sources.join(", ")} are mutually exclusive — use --embedder only (http(s):// URL or command)`);
|
|
40
|
+
}
|
|
41
|
+
if (embedder)
|
|
42
|
+
return parseEmbedderSpec(embedder);
|
|
43
|
+
if (url)
|
|
44
|
+
return { kind: "http", url };
|
|
45
|
+
if (cmd)
|
|
46
|
+
return { kind: "cmd", command: cmd };
|
|
47
|
+
return { kind: "none" };
|
|
48
|
+
}
|
|
49
|
+
export function describeEmbedConfig(c) {
|
|
50
|
+
switch (c.kind) {
|
|
51
|
+
case "http":
|
|
52
|
+
return `http ${c.url}`;
|
|
53
|
+
case "cmd":
|
|
54
|
+
return `cmd ${c.command}`;
|
|
55
|
+
case "none":
|
|
56
|
+
return "off";
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Instantiate a hook from a resolved config. Returns null if the
|
|
61
|
+
* config is `none` — callers decide whether that is fatal.
|
|
62
|
+
*/
|
|
63
|
+
export function createHookFromConfig(c) {
|
|
64
|
+
switch (c.kind) {
|
|
65
|
+
case "http":
|
|
66
|
+
return createHttpEmbedHook({ url: c.url });
|
|
67
|
+
case "cmd": {
|
|
68
|
+
// The command string is split on whitespace (shell-lite). If a
|
|
69
|
+
// user needs quoted args they can point --embedder at a wrapper
|
|
70
|
+
// script — same rule production sidecar deployments already use.
|
|
71
|
+
const parts = c.command.split(/\s+/).filter((p) => p.length > 0);
|
|
72
|
+
const [command, ...args] = parts;
|
|
73
|
+
if (!command)
|
|
74
|
+
throw new Error("--embedder command is empty");
|
|
75
|
+
return createCmdEmbedHook({ command, args });
|
|
76
|
+
}
|
|
77
|
+
case "none":
|
|
78
|
+
return null;
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
/** Commander option bundle for commands that accept an embedding provider. */
|
|
82
|
+
export const EMBED_CLI_OPTIONS = {
|
|
83
|
+
embedder: {
|
|
84
|
+
flags: "--embedder <spec>",
|
|
85
|
+
description: "embedding provider: command, or http(s):// URL",
|
|
86
|
+
},
|
|
87
|
+
embedUrl: {
|
|
88
|
+
flags: "--embed-url <url>",
|
|
89
|
+
description: "(deprecated) use --embedder with an http(s):// URL",
|
|
90
|
+
},
|
|
91
|
+
embedCmd: {
|
|
92
|
+
flags: "--embed-cmd <cmd>",
|
|
93
|
+
description: "(deprecated) use --embedder with a command",
|
|
94
|
+
},
|
|
95
|
+
};
|
|
96
|
+
export function embedFlagInputsFromCli(opts) {
|
|
97
|
+
return {
|
|
98
|
+
embedder: opts.embedder,
|
|
99
|
+
embed_url: opts.embedUrl,
|
|
100
|
+
embed_cmd: opts.embedCmd,
|
|
101
|
+
};
|
|
102
|
+
}
|
|
103
|
+
export function addEmbedCliOptions(cmd) {
|
|
104
|
+
return cmd
|
|
105
|
+
.option(EMBED_CLI_OPTIONS.embedder.flags, EMBED_CLI_OPTIONS.embedder.description)
|
|
106
|
+
.option(EMBED_CLI_OPTIONS.embedUrl.flags, EMBED_CLI_OPTIONS.embedUrl.description)
|
|
107
|
+
.option(EMBED_CLI_OPTIONS.embedCmd.flags, EMBED_CLI_OPTIONS.embedCmd.description);
|
|
108
|
+
}
|
|
109
|
+
//# sourceMappingURL=config.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/embed/config.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,kBAAkB,CAAC;AAC9C,OAAO,EAAE,kBAAkB,EAAE,MAAM,eAAe,CAAC;AAEnD,OAAO,EAAE,mBAAmB,EAAE,MAAM,gBAAgB,CAAC;AAgBrD;;;GAGG;AACH,MAAM,UAAU,iBAAiB,CAAC,KAAa;IAC7C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC7B,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,MAAM,IAAI,KAAK,CAAC,qBAAqB,CAAC,CAAC;IACzC,CAAC;IACD,IAAI,OAAO,CAAC,UAAU,CAAC,SAAS,CAAC,IAAI,OAAO,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QACpE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,OAAO,EAAE,CAAC;IACxC,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,OAAO,EAAE,CAAC;AAC3C,CAAC;AAED,MAAM,UAAU,kBAAkB,CAAC,SAA0B,EAAE;IAC7D,MAAM,GAAG,GAAG,UAAU,EAAE,CAAC;IAEzB,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,OAAO,CAAC,GAAG,CAAC,eAAe,IAAI,GAAG,CAAC,QAAQ,CAAC;IAChF,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,GAAG,CAAC,SAAS,CAAC;IAC9E,MAAM,GAAG,GAAG,MAAM,CAAC,SAAS,IAAI,OAAO,CAAC,GAAG,CAAC,gBAAgB,IAAI,GAAG,CAAC,SAAS,CAAC;IAE9E,MAAM,OAAO,GAAa,EAAE,CAAC;IAC7B,IAAI,QAAQ;QAAE,OAAO,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;IACzC,IAAI,GAAG;QAAE,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACrC,IAAI,GAAG;QAAE,OAAO,CAAC,IAAI,CAAC,aAAa,CAAC,CAAC;IACrC,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACvB,MAAM,IAAI,KAAK,CACb,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,2EAA2E,CACjG,CAAC;IACJ,CAAC;IAED,IAAI,QAAQ;QAAE,OAAO,iBAAiB,CAAC,QAAQ,CAAC,CAAC;IACjD,IAAI,GAAG;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,GAAG,EAAE,CAAC;IACtC,IAAI,GAAG;QAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,OAAO,EAAE,GAAG,EAAE,CAAC;IAC9C,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAC1B,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,CAAc;IAChD,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;QACf,KAAK,MAAM;YACT,OAAO,QAAQ,CAAC,CAAC,GAAG,EAAE,CAAC;QACzB,KAAK,KAAK;YACR,OAAO,OAAO,CAAC,CAAC,OAAO,EAAE,CAAC;QAC5B,KAAK,MAAM;YACT,OAAO,KAAK,CAAC;IACjB,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,CAAc;IACjD,QAAQ,CAAC,CAAC,IAAI,EAAE,CAAC;QACf,KAAK,MAAM;YACT,OAAO,mBAAmB,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC;QAC7C,KAAK,KAAK,CAAC,CAAC,CAAC;YACX,+DAA+D;YAC/D,gEAAgE;YAChE,iEAAiE;YACjE,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;YACjE,MAAM,CAAC,OAAO,EAAE,GAAG,IAAI,CAAC,GAAG,KAAK,CAAC;YACjC,IAAI,CAAC,OAAO;gBAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;YAC7D,OAAO,kBAAkB,CAAC,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC/C,CAAC;QACD,KAAK,MAAM;YACT,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,MAAM,CAAC,MAAM,iBAAiB,GAAG;IAC/B,QAAQ,EAAE;QACR,KAAK,EAAE,mBAAmB;QAC1B,WAAW,EAAE,gDAAgD;KAC9D;IACD,QAAQ,EAAE;QACR,KAAK,EAAE,mBAAmB;QAC1B,WAAW,EAAE,oDAAoD;KAClE;IACD,QAAQ,EAAE;QACR,KAAK,EAAE,mBAAmB;QAC1B,WAAW,EAAE,4CAA4C;KAC1D;CACO,CAAC;AAQX,MAAM,UAAU,sBAAsB,CAAC,IAAkB;IACvD,OAAO;QACL,QAAQ,EAAE,IAAI,CAAC,QAAQ;QACvB,SAAS,EAAE,IAAI,CAAC,QAAQ;QACxB,SAAS,EAAE,IAAI,CAAC,QAAQ;KACzB,CAAC;AACJ,CAAC;AAID,MAAM,UAAU,kBAAkB,CAAC,GAAY;IAC7C,OAAO,GAAG;SACP,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,KAAK,EAAE,iBAAiB,CAAC,QAAQ,CAAC,WAAW,CAAC;SAChF,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,KAAK,EAAE,iBAAiB,CAAC,QAAQ,CAAC,WAAW,CAAC;SAChF,MAAM,CAAC,iBAAiB,CAAC,QAAQ,CAAC,KAAK,EAAE,iBAAiB,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;AACtF,CAAC"}
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding hook contract (design §5.3).
|
|
3
|
+
*
|
|
4
|
+
* embed(chunks: string[]) → { vectors: number[][], model: string, dim: int }
|
|
5
|
+
*
|
|
6
|
+
* mrplex does NOT call any provider itself: the operator wires a hook
|
|
7
|
+
* that speaks this contract. m4-plan ships two shapes: HTTP endpoint
|
|
8
|
+
* (--embed-url) and long-lived subprocess with JSON-lines over stdio
|
|
9
|
+
* (--embed-cmd). In-process plugin is deferred (m4-plan §5 decision 1).
|
|
10
|
+
*
|
|
11
|
+
* Contract-validation happens here so every shape gets the same
|
|
12
|
+
* checks: batch alignment, per-vector dim, non-empty model. A single
|
|
13
|
+
* bad response fails the whole batch — never partial-write.
|
|
14
|
+
*/
|
|
15
|
+
export type EmbedResponse = {
|
|
16
|
+
vectors: number[][];
|
|
17
|
+
model: string;
|
|
18
|
+
dim: number;
|
|
19
|
+
};
|
|
20
|
+
export type EmbedHook = {
|
|
21
|
+
/** Human-facing description (URL / command), used in log lines. */
|
|
22
|
+
label: string;
|
|
23
|
+
/**
|
|
24
|
+
* Embed a batch. Batching, rate limiting, provider retries all live
|
|
25
|
+
* inside the hook (§5.3) — the worker only paces dispatch.
|
|
26
|
+
*
|
|
27
|
+
* Contract-invariant: `vectors[i]` corresponds to `chunks[i]`.
|
|
28
|
+
*/
|
|
29
|
+
embed(chunks: readonly string[]): Promise<EmbedResponse>;
|
|
30
|
+
/** Release resources (kill subprocess, close sockets). */
|
|
31
|
+
close(): Promise<void>;
|
|
32
|
+
};
|
|
33
|
+
/**
|
|
34
|
+
* Validate a response against the hook contract. Returns the response
|
|
35
|
+
* unchanged on success; throws a descriptive Error on any violation.
|
|
36
|
+
* Hooks call this before returning so the worker can trust the shape.
|
|
37
|
+
*/
|
|
38
|
+
export declare function validateEmbedResponse(resp: unknown, requested: readonly string[]): EmbedResponse;
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding hook contract (design §5.3).
|
|
3
|
+
*
|
|
4
|
+
* embed(chunks: string[]) → { vectors: number[][], model: string, dim: int }
|
|
5
|
+
*
|
|
6
|
+
* mrplex does NOT call any provider itself: the operator wires a hook
|
|
7
|
+
* that speaks this contract. m4-plan ships two shapes: HTTP endpoint
|
|
8
|
+
* (--embed-url) and long-lived subprocess with JSON-lines over stdio
|
|
9
|
+
* (--embed-cmd). In-process plugin is deferred (m4-plan §5 decision 1).
|
|
10
|
+
*
|
|
11
|
+
* Contract-validation happens here so every shape gets the same
|
|
12
|
+
* checks: batch alignment, per-vector dim, non-empty model. A single
|
|
13
|
+
* bad response fails the whole batch — never partial-write.
|
|
14
|
+
*/
|
|
15
|
+
/**
|
|
16
|
+
* Validate a response against the hook contract. Returns the response
|
|
17
|
+
* unchanged on success; throws a descriptive Error on any violation.
|
|
18
|
+
* Hooks call this before returning so the worker can trust the shape.
|
|
19
|
+
*/
|
|
20
|
+
export function validateEmbedResponse(resp, requested) {
|
|
21
|
+
if (typeof resp !== "object" || resp === null) {
|
|
22
|
+
throw new Error("embed hook: response is not an object");
|
|
23
|
+
}
|
|
24
|
+
const r = resp;
|
|
25
|
+
const model = r.model;
|
|
26
|
+
if (typeof model !== "string" || model.length === 0) {
|
|
27
|
+
throw new Error("embed hook: response.model must be a non-empty string");
|
|
28
|
+
}
|
|
29
|
+
const dim = r.dim;
|
|
30
|
+
if (typeof dim !== "number" || !Number.isInteger(dim) || dim <= 0) {
|
|
31
|
+
throw new Error(`embed hook: response.dim must be a positive integer (got ${JSON.stringify(dim)})`);
|
|
32
|
+
}
|
|
33
|
+
const vectors = r.vectors;
|
|
34
|
+
if (!Array.isArray(vectors)) {
|
|
35
|
+
throw new Error("embed hook: response.vectors must be an array");
|
|
36
|
+
}
|
|
37
|
+
if (vectors.length !== requested.length) {
|
|
38
|
+
throw new Error(`embed hook: vectors.length (${vectors.length}) !== requested chunks (${requested.length})`);
|
|
39
|
+
}
|
|
40
|
+
for (let i = 0; i < vectors.length; i++) {
|
|
41
|
+
const v = vectors[i];
|
|
42
|
+
if (!Array.isArray(v)) {
|
|
43
|
+
throw new Error(`embed hook: vectors[${i}] is not an array`);
|
|
44
|
+
}
|
|
45
|
+
if (v.length !== dim) {
|
|
46
|
+
throw new Error(`embed hook: vectors[${i}].length (${v.length}) !== dim (${dim})`);
|
|
47
|
+
}
|
|
48
|
+
for (let j = 0; j < v.length; j++) {
|
|
49
|
+
if (typeof v[j] !== "number" || !Number.isFinite(v[j])) {
|
|
50
|
+
throw new Error(`embed hook: vectors[${i}][${j}] is not a finite number`);
|
|
51
|
+
}
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return { vectors: vectors, model, dim };
|
|
55
|
+
}
|
|
56
|
+
//# sourceMappingURL=hook.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"hook.js","sourceRoot":"","sources":["../../src/embed/hook.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;GAaG;AAsBH;;;;GAIG;AACH,MAAM,UAAU,qBAAqB,CAAC,IAAa,EAAE,SAA4B;IAC/E,IAAI,OAAO,IAAI,KAAK,QAAQ,IAAI,IAAI,KAAK,IAAI,EAAE,CAAC;QAC9C,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,CAAC,GAAG,IAA+B,CAAC;IAC1C,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;IACtB,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CAAC,uDAAuD,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,GAAG,GAAG,CAAC,CAAC,GAAG,CAAC;IAClB,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,GAAG,IAAI,CAAC,EAAE,CAAC;QAClE,MAAM,IAAI,KAAK,CACb,4DAA4D,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CACnF,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,CAAC,CAAC,OAAO,CAAC;IAC1B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;QAC5B,MAAM,IAAI,KAAK,CAAC,+CAA+C,CAAC,CAAC;IACnE,CAAC;IACD,IAAI,OAAO,CAAC,MAAM,KAAK,SAAS,CAAC,MAAM,EAAE,CAAC;QACxC,MAAM,IAAI,KAAK,CACb,+BAA+B,OAAO,CAAC,MAAM,2BAA2B,SAAS,CAAC,MAAM,GAAG,CAC5F,CAAC;IACJ,CAAC;IACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QACxC,MAAM,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;QACrB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;YACtB,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,mBAAmB,CAAC,CAAC;QAC/D,CAAC;QACD,IAAI,CAAC,CAAC,MAAM,KAAK,GAAG,EAAE,CAAC;YACrB,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,aAAa,CAAC,CAAC,MAAM,cAAc,GAAG,GAAG,CAAC,CAAC;QACrF,CAAC;QACD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YAClC,IAAI,OAAO,CAAC,CAAC,CAAC,CAAC,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;gBACvD,MAAM,IAAI,KAAK,CAAC,uBAAuB,CAAC,KAAK,CAAC,0BAA0B,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,EAAE,OAAO,EAAE,OAAqB,EAAE,KAAK,EAAE,GAAG,EAAE,CAAC;AACxD,CAAC"}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HTTP embedding hook (--embed-url). POST { chunks: string[] } → the
|
|
3
|
+
* contract response (design §5.3, m4-plan §5 decision 1).
|
|
4
|
+
*
|
|
5
|
+
* Uses Node ≥ 20's built-in fetch — no dep. Timeout defaults to 30s;
|
|
6
|
+
* anything longer usually means the operator wants a subprocess hook.
|
|
7
|
+
*/
|
|
8
|
+
import { type EmbedHook } from "./hook.js";
|
|
9
|
+
export type HttpHookOptions = {
|
|
10
|
+
url: string;
|
|
11
|
+
timeoutMs?: number;
|
|
12
|
+
};
|
|
13
|
+
export declare function createHttpEmbedHook(opts: HttpHookOptions): EmbedHook;
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* HTTP embedding hook (--embed-url). POST { chunks: string[] } → the
|
|
3
|
+
* contract response (design §5.3, m4-plan §5 decision 1).
|
|
4
|
+
*
|
|
5
|
+
* Uses Node ≥ 20's built-in fetch — no dep. Timeout defaults to 30s;
|
|
6
|
+
* anything longer usually means the operator wants a subprocess hook.
|
|
7
|
+
*/
|
|
8
|
+
import { validateEmbedResponse } from "./hook.js";
|
|
9
|
+
export function createHttpEmbedHook(opts) {
|
|
10
|
+
const timeout = opts.timeoutMs ?? 30_000;
|
|
11
|
+
return {
|
|
12
|
+
label: opts.url,
|
|
13
|
+
async embed(chunks) {
|
|
14
|
+
const controller = new AbortController();
|
|
15
|
+
const timer = setTimeout(() => controller.abort(), timeout);
|
|
16
|
+
try {
|
|
17
|
+
const res = await fetch(opts.url, {
|
|
18
|
+
method: "POST",
|
|
19
|
+
headers: { "content-type": "application/json" },
|
|
20
|
+
body: JSON.stringify({ chunks }),
|
|
21
|
+
signal: controller.signal,
|
|
22
|
+
});
|
|
23
|
+
if (!res.ok) {
|
|
24
|
+
const text = await res.text().catch(() => "");
|
|
25
|
+
throw new Error(`embed hook HTTP ${res.status}: ${text.slice(0, 500)}`);
|
|
26
|
+
}
|
|
27
|
+
const body = await res.json();
|
|
28
|
+
return validateEmbedResponse(body, chunks);
|
|
29
|
+
}
|
|
30
|
+
finally {
|
|
31
|
+
clearTimeout(timer);
|
|
32
|
+
}
|
|
33
|
+
},
|
|
34
|
+
async close() {
|
|
35
|
+
// fetch is stateless; nothing to close.
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
//# sourceMappingURL=http-hook.js.map
|