@functional-systems/srcx 0.1.0-dev.7 → 0.1.0-dev.8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +5 -0
- package/DESIGN.md +55 -21
- package/README.md +50 -1
- package/VALIDATION.md +45 -0
- package/dist/build.d.ts +68 -0
- package/dist/build.js +106 -19
- package/dist/build.js.map +1 -1
- package/dist/cli.js +20 -7
- package/dist/cli.js.map +1 -1
- package/dist/publish.js +11 -5
- package/dist/publish.js.map +1 -1
- package/dist/remote.js +30 -3
- package/dist/remote.js.map +1 -1
- package/dist/repository.d.ts +4 -1
- package/dist/repository.js +15 -11
- package/dist/repository.js.map +1 -1
- package/dist/search.d.ts +7 -2
- package/dist/search.js +24 -4
- package/dist/search.js.map +1 -1
- package/package.json +2 -2
- package/src/build.ts +138 -22
- package/src/cli.ts +59 -10
- package/src/publish.ts +18 -6
- package/src/remote.ts +44 -2
- package/src/repository.ts +20 -11
- package/src/search.ts +37 -8
package/dist/search.js
CHANGED
|
@@ -3,6 +3,7 @@ import { randomUUID } from "node:crypto";
|
|
|
3
3
|
import { atomic, hash, invariant, json, lineAt } from "./common.js";
|
|
4
4
|
import { one, tag } from "./remote.js";
|
|
5
5
|
import { stateRoot } from "./settings.js";
|
|
6
|
+
import { PRESET, recordHash } from "./build.js";
|
|
6
7
|
export async function inventoryAt(store, r, v) {
|
|
7
8
|
const current = (await store.tags()).find((t) => t.name === v.tagName);
|
|
8
9
|
invariant(current?.snapshotId === v.snapshotId, "Pinned Tag was deleted or recreated.");
|
|
@@ -45,12 +46,31 @@ export function lexicalQuery(query, filters = {}) {
|
|
|
45
46
|
],
|
|
46
47
|
};
|
|
47
48
|
}
|
|
48
|
-
export
|
|
49
|
+
export function retrievalQuery(query, size, filters, mode) {
|
|
50
|
+
invariant(["lexical", "semantic", "hybrid"].includes(mode), "Unknown search mode.");
|
|
51
|
+
const lexical = lexicalQuery(query, filters);
|
|
52
|
+
if (mode === "lexical")
|
|
53
|
+
return lexical;
|
|
54
|
+
const semantic = {
|
|
55
|
+
knn: {
|
|
56
|
+
field: "embedding",
|
|
57
|
+
queryText: query,
|
|
58
|
+
k: size,
|
|
59
|
+
filter: {
|
|
60
|
+
bool: lexical.bool.filter((c) => c.occur === "filter"),
|
|
61
|
+
},
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
return mode === "semantic" ? semantic : { rrf: [lexical, semantic] };
|
|
65
|
+
}
|
|
66
|
+
export async function search(store, s, r, v, query, size = 10, filters = {}, mode = "lexical") {
|
|
49
67
|
invariant(Number.isInteger(size) && size > 0 && size <= 100, "Limit must be an integer from 1 to 100.");
|
|
68
|
+
invariant(mode === "lexical" || r.preset?.embedding?.managed, "Semantic/hybrid search requires a managed embedding Collection; register with --embedding text-embedding-3-small.");
|
|
69
|
+
const request = retrievalQuery(query, size, filters, mode);
|
|
50
70
|
const entries = await inventoryAt(store, r, v);
|
|
51
71
|
const files = new Map(entries.filter((e) => e.status === "included").map((e) => [e.fileId, e]));
|
|
52
72
|
const result = [];
|
|
53
|
-
for (const hit of await store.query(v.tagName,
|
|
73
|
+
for (const hit of await store.query(v.tagName, request, size)) {
|
|
54
74
|
const d = hit.doc, e = files.get(d.fileId);
|
|
55
75
|
invariant(d.kind === "chunk" &&
|
|
56
76
|
d.configHash === r.configHash &&
|
|
@@ -68,7 +88,7 @@ export async function search(store, s, r, v, query, size = 10, filters = {}) {
|
|
|
68
88
|
path: String(d.path),
|
|
69
89
|
contentHash: String(d.contentHash),
|
|
70
90
|
chunkId: d.id,
|
|
71
|
-
chunkHash:
|
|
91
|
+
chunkHash: recordHash(d, r.preset ?? PRESET),
|
|
72
92
|
startLine: Number(d.startLine),
|
|
73
93
|
endLine: Number(d.endLine),
|
|
74
94
|
};
|
|
@@ -132,7 +152,7 @@ export async function readHandle(store, s, h, options = {}) {
|
|
|
132
152
|
if (h.chunkId) {
|
|
133
153
|
const d = await one(store, tag(v.tagName), h.chunkId);
|
|
134
154
|
invariant(d &&
|
|
135
|
-
|
|
155
|
+
recordHash(d, h.repository.preset ?? PRESET) === h.chunkHash &&
|
|
136
156
|
d.fileId === h.fileId &&
|
|
137
157
|
d.contentHash === h.contentHash, "Pinned chunk has changed or is missing.");
|
|
138
158
|
const a = Number(d.startByte), z = Number(d.endByte);
|
package/dist/search.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAY,MAAM,aAAa,CAAC;AAC9E,OAAO,EAAE,GAAG,EAAE,GAAG,EAAwB,MAAM,aAAa,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAiB,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAY,MAAM,aAAa,CAAC;AAC9E,OAAO,EAAE,GAAG,EAAE,GAAG,EAAwB,MAAM,aAAa,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAiB,MAAM,eAAe,CAAC;AAGzD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAsB,MAAM,YAAY,CAAC;AAepE,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAsB,EACtB,CAAa,EACb,CAAY;IAEZ,MAAM,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;IACvE,SAAS,CACP,OAAO,EAAE,UAAU,KAAK,CAAC,CAAC,UAAU,EACpC,sCAAsC,CACvC,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;IAC9D,SAAS,CACP,IAAI,EAAE,MAAM,KAAK,CAAC,CAAC,MAAM;QACvB,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO;QAC1B,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;QAChC,IAAI,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS;QAC9B,IAAI,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW;QAClC,IAAI,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa,EACxC,oCAAoC,CACrC,CAAC;IACF,SAAS,CACP,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,EACpC,sCAAsC,CACvC,CAAC;IACF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,IAAK,IAAI,CAAC,gBAA6B,CAAC,OAAO,EAAE,EAAE,CAAC;QACpE,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;QAClD,SAAS,CACP,IAAI,EAAE,IAAI,KAAK,WAAW;YACxB,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;YAChC,IAAI,CAAC,WAAW,KAAK,CAAC;YACtB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,QAAQ,EACtC,yBAAyB,CAC1B,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,GAAI,IAAI,CAAC,OAA2B,CAAC,CAAC;IACrD,CAAC;IACD,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,CAAC;IAC5E,OAAO,OAAO,CAAC;AACjB,CAAC;AACD,MAAM,UAAU,YAAY,CAC1B,KAAa,EACb,UAAgD,EAAE;IAElD,SAAS,CACP,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,EAC/C,uCAAuC,CACxC,CAAC;IACF,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,KAAa,EAAE,EAAE,CAAC,CAAC;QACjD,WAAW,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE;QACpE,KAAK,EAAE,QAAQ;KAChB,CAAC,CAAC;IACH,OAAO;QACL,IAAI,EAAE;YACJ,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC;YACxB;gBACE,WAAW,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,IAAI,EAAE;gBACpE,KAAK,EAAE,MAAM;aACd;YACD,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,KAAa,EACb,IAAY,EACZ,OAA6C,EAC7C,IAAgB;IAEhB,SAAS,CACP,CAAC,SAAS,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAChD,sBAAsB,CACvB,CAAC;IACF,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC7C,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC;IACvC,MAAM,QAAQ,GAAG;QACf,GAAG,EAAE;YACH,KAAK,EAAE,WAAW;YAClB,SAAS,EAAE,KAAK;YAChB,CAAC,EAAE,IAAI;YACP,MAAM,EAAE;gBACN,IAAI,EAAG,OAAO,CAAC,IAA4B,CAAC,MAAM,CAChD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,QAAQ,CAC5B;aACF;SACF;KACF,CAAC;IACF,OAAO,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;AACvE,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAsB,EACtB,CAAW,EACX,CAAa,EACb,CAAY,EACZ,KAAa,EACb,IAAI,GAAG,EAAE,EACT,UAAgD,EAAE,EAClD,OAAmB,SAAS;IAE5B,SAAS,CACP,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,GAAG,EACjD,yCAAyC,CAC1C,CAAC;IACF,SAAS,CACP,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAClD,mHAAmH,CACpH,CAAC;IACF,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,IAAI,GAAG,CACnB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CACzE,CAAC;IACF,MAAM,MAAM,GAAG,EAAE,CAAC;IAClB,KAAK,MAAM,GAAG,IAAI,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,EACf,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAgB,CAAC,CAAC;QACpC,SAAS,CACP,CAAC,CAAC,IAAI,KAAK,OAAO;YAChB,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;YAC7B,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3B,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW;YAC/B,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EACnB,qDAAqD,CACtD,CAAC;QACF,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;QACxB,MAAM,MAAM,GAAW;YACrB,EAAE;YACF,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YACxB,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;YACpB,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC;YAClC,OAAO,EAAE,CAAC,CAAC,EAAE;YACb,SAAS,EAAE,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC;YAC5C,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;YAC9B,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;SAC3B,CAAC;QACF,sEAAsE;QACtE,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACpD,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,EAAE,GAAG,OAAO,CAAC,EAAE,MAAM,CAAC,CAAC;QACjE,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,EAAE;YACZ,UAAU,EAAE,CAAC,CAAC,OAAO;YACrB,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YAC1C,QAAQ,EAAE,QAAQ,CAAC,QAAQ;SAC5B,CAAC,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAAU;IACzC,SAAS,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,2BAA2B,CAAC,CAAC;IACnE,MAAM,CAAC,GAAG,MAAM,IAAI,CAAS,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC;IACzE,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,yBAAyB,CAAC,CAAC;IAClD,OAAO,CAAC,CAAC;AACX,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAsB,EACtB,CAAW,EACX,CAAa,EACb,CAAY,EACZ,IAAY;IAEZ,MAAM,CAAC,GAAG,CAAC,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IACxE,SAAS,CACP,CAAC,EAAE,MAAM,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,WAAW,EACrD,sBAAsB,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAC1D,CAAC;IACF,OAAO;QACL,EAAE,EAAE,QAAQ;QACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,UAAU,EAAE,CAAC;QACb,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,IAAI;QACJ,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;KACX,CAAC;AACJ,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAsB,EACtB,CAAW,EACX,CAAS,EACT,UAII,EAAE;IAEN,SAAS,CACP,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,EACpD,qFAAqF,CACtF,CAAC;IACF,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EACjB,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;IACnE,SAAS,CACP,OAAO,EAAE,UAAU,KAAK,CAAC,CAAC,UAAU,EACpC,8CAA8C,CAC/C,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACxD,SAAS,CACP,IAAI,EAAE,IAAI,KAAK,MAAM;QACnB,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU,CAAC,UAAU;QAC3C,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI;QACpB,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ;QACnC,IAAI,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW;QAClC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,EACtD,wDAAwD,CACzD,CAAC;IACF,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACzC,MAAM,KAAK,GACT,GAAG,CAAC,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;QACtD,SAAS,CACP,CAAC;YACC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS;YAC5D,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;YACrB,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW,EACjC,yCAAyC,CAC1C,CAAC;QACF,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,EAC3B,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACxB,SAAS,CACP,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;YACjB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;YACnB,CAAC,IAAI,CAAC;YACN,CAAC,GAAG,CAAC;YACL,CAAC,IAAI,GAAG,CAAC,MAAM;YACf,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS;YAC9B,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAClC,uBAAuB,CACxB,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;IACrC,SAAS,CACP,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,EACzC,wCAAwC,CACzC,CAAC;IACF,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAC3C,GAAG,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC;IACxC,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,KAAK,GAAG,CAAC,CAAC;QACV,GAAG,GAAG,KAAK,CAAC;IACd,CAAC;SAAM,CAAC;QACN,SAAS,CACP,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;YACrB,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC;YACrB,KAAK,IAAI,CAAC;YACV,GAAG,IAAI,KAAK;YACZ,KAAK,IAAI,KAAK;YACd,GAAG,IAAI,KAAK,EACd,iCAAiC,CAClC,CAAC;QACF,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC;QACrC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC;IACvC,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ;QACjC,CAAC,CAAC,IAAI,CAAC,UAAU;QACjB,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,OAAO;QACL,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,OAAO;QAChC,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,SAAS,EAAE,KAAK;QAChB,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,UAAU;QACV,QAAQ,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,IAAI,IAAI,KAAK,IAAI,GAAG,EAAE;KAC7E,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@functional-systems/srcx",
|
|
3
|
-
"version": "0.1.0-dev.
|
|
3
|
+
"version": "0.1.0-dev.8",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Version-aware code search on LambdaDB",
|
|
6
6
|
"type": "module",
|
|
@@ -60,5 +60,5 @@
|
|
|
60
60
|
"url": "https://github.com/lambdadb/srcx/issues"
|
|
61
61
|
},
|
|
62
62
|
"license": "Apache-2.0",
|
|
63
|
-
"gitHead": "
|
|
63
|
+
"gitHead": "163c8a96239f626aecadc3ec8aea3ba39bbc2feb"
|
|
64
64
|
}
|
package/src/build.ts
CHANGED
|
@@ -30,7 +30,14 @@ export type Preset = {
|
|
|
30
30
|
/** Internal evaluation override; the CLI keeps its original enrichment. */
|
|
31
31
|
enrichment?: "path-only-v1";
|
|
32
32
|
maxFileBytes: number;
|
|
33
|
-
embedding: {
|
|
33
|
+
embedding: {
|
|
34
|
+
model: string;
|
|
35
|
+
dimensions: number;
|
|
36
|
+
managed?: true;
|
|
37
|
+
provider?: "openai";
|
|
38
|
+
similarity?: "cosine";
|
|
39
|
+
sourceField?: "embeddingText";
|
|
40
|
+
} | null;
|
|
34
41
|
};
|
|
35
42
|
export const PRESET: Preset = {
|
|
36
43
|
schemaVersion: 1,
|
|
@@ -39,6 +46,63 @@ export const PRESET: Preset = {
|
|
|
39
46
|
maxFileBytes: 1024 * 1024,
|
|
40
47
|
embedding: null,
|
|
41
48
|
};
|
|
49
|
+
/** Opt-in managed preset; PRESET stays byte-for-byte compatible with lexical corpora. */
|
|
50
|
+
export const MANAGED_PRESET: Preset = {
|
|
51
|
+
...PRESET,
|
|
52
|
+
embedding: {
|
|
53
|
+
managed: true,
|
|
54
|
+
provider: "openai",
|
|
55
|
+
model: "text-embedding-3-small",
|
|
56
|
+
dimensions: 1536,
|
|
57
|
+
similarity: "cosine",
|
|
58
|
+
sourceField: "embeddingText",
|
|
59
|
+
},
|
|
60
|
+
};
|
|
61
|
+
export function presetFor(embedding = "none"): Preset {
|
|
62
|
+
invariant(
|
|
63
|
+
["none", "text-embedding-3-small"].includes(embedding),
|
|
64
|
+
"Embedding must be none or text-embedding-3-small.",
|
|
65
|
+
);
|
|
66
|
+
return embedding === "none" ? PRESET : MANAGED_PRESET;
|
|
67
|
+
}
|
|
68
|
+
export function supportedPreset(preset: unknown): preset is Preset {
|
|
69
|
+
return (
|
|
70
|
+
preset !== undefined &&
|
|
71
|
+
[hash(PRESET), hash(MANAGED_PRESET)].includes(hash(preset))
|
|
72
|
+
);
|
|
73
|
+
}
|
|
74
|
+
export function indexConfigs(preset: Preset = PRESET) {
|
|
75
|
+
if (!preset.embedding?.managed) return INDEX_CONFIGS;
|
|
76
|
+
const { managed: _, ...embedding } = preset.embedding;
|
|
77
|
+
return {
|
|
78
|
+
...INDEX_CONFIGS,
|
|
79
|
+
embeddingText: { type: "text", analyzers: ["standard"] },
|
|
80
|
+
embedding: { type: "vector", managedEmbedding: true, embedding },
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
/** Validate server enrichment separately; all other payload fields remain exact. */
|
|
84
|
+
export function recordHash(doc: Doc, preset: Preset): string {
|
|
85
|
+
if (!preset.embedding?.managed) return hash(doc);
|
|
86
|
+
const { embedding, ...source } = doc;
|
|
87
|
+
if (doc.kind === "chunk" && doc.embeddingStatus === "managed") {
|
|
88
|
+
invariant(
|
|
89
|
+
typeof doc.embeddingText === "string" &&
|
|
90
|
+
doc.embeddingText === doc.searchText &&
|
|
91
|
+
doc.embeddingInputHash ===
|
|
92
|
+
hash([preset.embedding, doc.embeddingText]) &&
|
|
93
|
+
Array.isArray(embedding) &&
|
|
94
|
+
embedding.length === preset.embedding.dimensions &&
|
|
95
|
+
embedding.every((v) => typeof v === "number" && Number.isFinite(v)) &&
|
|
96
|
+
embedding.some((v) => v !== 0),
|
|
97
|
+
"Missing or invalid managed embedding.",
|
|
98
|
+
);
|
|
99
|
+
} else
|
|
100
|
+
invariant(
|
|
101
|
+
embedding === undefined && doc.embeddingText === undefined,
|
|
102
|
+
"Unexpected managed embedding on an ineligible record.",
|
|
103
|
+
);
|
|
104
|
+
return hash(source);
|
|
105
|
+
}
|
|
42
106
|
export type InventoryItem = {
|
|
43
107
|
path: string;
|
|
44
108
|
pathBase64: string;
|
|
@@ -54,6 +118,8 @@ export type InventoryItem = {
|
|
|
54
118
|
parseStatus?: string;
|
|
55
119
|
tokens?: number;
|
|
56
120
|
embedded?: number;
|
|
121
|
+
managed?: number;
|
|
122
|
+
managedTokens?: number;
|
|
57
123
|
skipped?: number;
|
|
58
124
|
};
|
|
59
125
|
export type Build = {
|
|
@@ -152,6 +218,7 @@ export async function materialize(args: {
|
|
|
152
218
|
}
|
|
153
219
|
invariant(
|
|
154
220
|
!preset.embedding ||
|
|
221
|
+
(preset.embedding.managed && supportedPreset(preset)) ||
|
|
155
222
|
(args.embedder?.model === preset.embedding.model &&
|
|
156
223
|
args.embedder.dimensions === preset.embedding.dimensions),
|
|
157
224
|
"Embedding provider must match the pinned preset.",
|
|
@@ -296,31 +363,40 @@ export async function materialize(args: {
|
|
|
296
363
|
const eligible = !["imports", "structural"].includes(span.chunkKind);
|
|
297
364
|
if (preset.embedding && eligible) {
|
|
298
365
|
const inputHash = hash([preset.embedding, span.searchText]);
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
366
|
+
if (preset.embedding.managed) {
|
|
367
|
+
doc.embeddingText = span.searchText;
|
|
368
|
+
doc.embeddingInputHash = inputHash;
|
|
369
|
+
doc.embeddingStatus = "managed";
|
|
370
|
+
delete doc.embeddingSkipReason;
|
|
371
|
+
item.managed = (item.managed ?? 0) + 1;
|
|
372
|
+
item.managedTokens = (item.managedTokens ?? 0) + span.tokenCount;
|
|
373
|
+
} else {
|
|
374
|
+
const cachePath = args.cache
|
|
375
|
+
? join(args.cache, inputHash + ".json")
|
|
376
|
+
: undefined;
|
|
377
|
+
let vector = cachePath
|
|
378
|
+
? await optionalJson<number[]>(cachePath)
|
|
379
|
+
: undefined;
|
|
380
|
+
if (!vector) {
|
|
381
|
+
vector = await args.embedder!.embed(span.searchText);
|
|
382
|
+
invariant(
|
|
383
|
+
vector.length === preset.embedding.dimensions &&
|
|
384
|
+
vector.every(Number.isFinite),
|
|
385
|
+
"Invalid embedding output.",
|
|
386
|
+
);
|
|
387
|
+
if (cachePath) await atomic(cachePath, vector);
|
|
388
|
+
}
|
|
307
389
|
invariant(
|
|
308
390
|
vector.length === preset.embedding.dimensions &&
|
|
309
391
|
vector.every(Number.isFinite),
|
|
310
|
-
"Invalid
|
|
392
|
+
"Invalid cached vector.",
|
|
311
393
|
);
|
|
312
|
-
|
|
394
|
+
doc.embedding = vector;
|
|
395
|
+
doc.embeddingInputHash = inputHash;
|
|
396
|
+
doc.embeddingStatus = "embedded";
|
|
397
|
+
delete doc.embeddingSkipReason;
|
|
398
|
+
item.embedded!++;
|
|
313
399
|
}
|
|
314
|
-
invariant(
|
|
315
|
-
vector.length === preset.embedding.dimensions &&
|
|
316
|
-
vector.every(Number.isFinite),
|
|
317
|
-
"Invalid cached vector.",
|
|
318
|
-
);
|
|
319
|
-
doc.embedding = vector;
|
|
320
|
-
doc.embeddingInputHash = inputHash;
|
|
321
|
-
doc.embeddingStatus = "embedded";
|
|
322
|
-
delete doc.embeddingSkipReason;
|
|
323
|
-
item.embedded!++;
|
|
324
400
|
} else {
|
|
325
401
|
doc.embeddingSkipReason = preset.embedding
|
|
326
402
|
? span.chunkKind === "imports"
|
|
@@ -399,6 +475,15 @@ export async function materialize(args: {
|
|
|
399
475
|
excluded: items.filter((e) => e.status === "excluded").length,
|
|
400
476
|
chunks: items.reduce((n, e) => n + (e.chunkIds?.length ?? 0), 0),
|
|
401
477
|
tokens: items.reduce((n, e) => n + (e.tokens ?? 0), 0),
|
|
478
|
+
...(preset.embedding?.managed
|
|
479
|
+
? {
|
|
480
|
+
managed: items.reduce((n, e) => n + (e.managed ?? 0), 0),
|
|
481
|
+
managedTokens: items.reduce(
|
|
482
|
+
(n, e) => n + (e.managedTokens ?? 0),
|
|
483
|
+
0,
|
|
484
|
+
),
|
|
485
|
+
}
|
|
486
|
+
: {}),
|
|
402
487
|
embedded: items.reduce((n, e) => n + (e.embedded ?? 0), 0),
|
|
403
488
|
skipped: items.reduce((n, e) => n + (e.skipped ?? 0), 0),
|
|
404
489
|
},
|
|
@@ -450,6 +535,17 @@ export async function validateBuild(b: Build): Promise<void> {
|
|
|
450
535
|
doc.configHash === b.configHash && doc.schemaVersion === 1,
|
|
451
536
|
"Mixed configuration.",
|
|
452
537
|
);
|
|
538
|
+
if (b.preset.embedding?.managed) {
|
|
539
|
+
invariant(
|
|
540
|
+
doc.embedding === undefined,
|
|
541
|
+
"Managed artifacts must not contain vectors.",
|
|
542
|
+
);
|
|
543
|
+
if (doc.kind !== "chunk")
|
|
544
|
+
invariant(
|
|
545
|
+
doc.embeddingText === undefined,
|
|
546
|
+
"Only eligible chunks can contain embedding input.",
|
|
547
|
+
);
|
|
548
|
+
}
|
|
453
549
|
if (doc.kind === "file") {
|
|
454
550
|
finish();
|
|
455
551
|
activeFile = doc;
|
|
@@ -505,7 +601,27 @@ export async function validateBuild(b: Build): Promise<void> {
|
|
|
505
601
|
Number(doc.tokenCount) <= CHUNKER.maxTokens,
|
|
506
602
|
"Invalid chunk source/position/token binding.",
|
|
507
603
|
);
|
|
508
|
-
if (
|
|
604
|
+
if (b.preset.embedding?.managed) {
|
|
605
|
+
const eligible = !["imports", "structural"].includes(
|
|
606
|
+
String(doc.chunkKind),
|
|
607
|
+
);
|
|
608
|
+
invariant(
|
|
609
|
+
eligible
|
|
610
|
+
? doc.embeddingStatus === "managed" &&
|
|
611
|
+
doc.embeddingText === doc.searchText &&
|
|
612
|
+
doc.embeddingInputHash ===
|
|
613
|
+
hash([b.preset.embedding, doc.searchText]) &&
|
|
614
|
+
doc.embeddingSkipReason === undefined
|
|
615
|
+
: doc.embeddingStatus === "skipped" &&
|
|
616
|
+
doc.embeddingText === undefined &&
|
|
617
|
+
doc.embeddingInputHash === undefined &&
|
|
618
|
+
doc.embeddingSkipReason ===
|
|
619
|
+
(doc.chunkKind === "imports"
|
|
620
|
+
? "imports-only"
|
|
621
|
+
: "structural-only"),
|
|
622
|
+
"Invalid managed embedding input or eligibility.",
|
|
623
|
+
);
|
|
624
|
+
} else if (doc.embeddingStatus === "embedded")
|
|
509
625
|
invariant(
|
|
510
626
|
b.preset.embedding &&
|
|
511
627
|
Array.isArray(doc.embedding) &&
|
package/src/cli.ts
CHANGED
|
@@ -1,12 +1,20 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
2
|
import { readFileSync } from "node:fs";
|
|
3
|
-
import { Command, InvalidArgumentError } from "commander";
|
|
3
|
+
import { Command, InvalidArgumentError, Option } from "commander";
|
|
4
4
|
import { mkdir } from "node:fs/promises";
|
|
5
5
|
import { join, resolve } from "node:path";
|
|
6
6
|
import { randomUUID } from "node:crypto";
|
|
7
7
|
import { configure, loadSettings, stateRoot } from "./settings.js";
|
|
8
8
|
import { identity, resolveCommit } from "./git.js";
|
|
9
|
-
import {
|
|
9
|
+
import {
|
|
10
|
+
loadBuild,
|
|
11
|
+
materialize,
|
|
12
|
+
validateBuild,
|
|
13
|
+
type Build,
|
|
14
|
+
type Preset,
|
|
15
|
+
PRESET,
|
|
16
|
+
presetFor,
|
|
17
|
+
} from "./build.js";
|
|
10
18
|
import { invariant, optionalJson } from "./common.js";
|
|
11
19
|
import { LambdaRemote } from "./remote.js";
|
|
12
20
|
import {
|
|
@@ -87,6 +95,14 @@ repo
|
|
|
87
95
|
.option("--remote <name>")
|
|
88
96
|
.option("--description <text>")
|
|
89
97
|
.option("--tag <key=value>", "One optional Collection metadata tag")
|
|
98
|
+
.addOption(
|
|
99
|
+
new Option(
|
|
100
|
+
"--embedding <model>",
|
|
101
|
+
"Managed model (source text is sent to the provider on import)",
|
|
102
|
+
)
|
|
103
|
+
.choices(["none", "text-embedding-3-small"])
|
|
104
|
+
.default("none"),
|
|
105
|
+
)
|
|
90
106
|
.action(async (o) => {
|
|
91
107
|
const { remote } = await connected();
|
|
92
108
|
let labels: Record<string, string> | undefined;
|
|
@@ -101,6 +117,7 @@ repo
|
|
|
101
117
|
remote: o.remote,
|
|
102
118
|
description: o.description,
|
|
103
119
|
labels,
|
|
120
|
+
preset: presetFor(o.embedding),
|
|
104
121
|
}),
|
|
105
122
|
);
|
|
106
123
|
});
|
|
@@ -128,6 +145,12 @@ cli
|
|
|
128
145
|
.option("--repo <name>", "Registered remote repository")
|
|
129
146
|
.option("--ref <ref>", "Local branch, Git tag, or commit OID")
|
|
130
147
|
.option("--dry-run", "Only materialize and validate a local build")
|
|
148
|
+
.addOption(
|
|
149
|
+
new Option(
|
|
150
|
+
"--embedding <model>",
|
|
151
|
+
"Preset for --dry-run --path; connected imports use the repository preset",
|
|
152
|
+
).choices(["none", "text-embedding-3-small"]),
|
|
153
|
+
)
|
|
131
154
|
.option("--output <directory>", "New artifact directory")
|
|
132
155
|
.option(
|
|
133
156
|
"--previous <directory>",
|
|
@@ -140,6 +163,10 @@ cli
|
|
|
140
163
|
)
|
|
141
164
|
.option("--timeout <seconds>", "Validation wait deadline", integer, 120)
|
|
142
165
|
.action(async (o) => {
|
|
166
|
+
invariant(
|
|
167
|
+
!o.embedding || (o.dryRun && o.path),
|
|
168
|
+
"--embedding is only accepted with --dry-run --path; connected imports use the repository preset.",
|
|
169
|
+
);
|
|
143
170
|
invariant(!(o.path && o.repo), "Choose --path or --repo.");
|
|
144
171
|
invariant(
|
|
145
172
|
!o.previous || o.dryRun,
|
|
@@ -154,6 +181,7 @@ cli
|
|
|
154
181
|
source: Awaited<ReturnType<typeof identity>>,
|
|
155
182
|
previous?: Build,
|
|
156
183
|
state?: string,
|
|
184
|
+
preset: Preset = PRESET,
|
|
157
185
|
) => {
|
|
158
186
|
invariant(o.ref, "--ref is required to build a commit.");
|
|
159
187
|
const commit = await resolveCommit(source.path, o.ref);
|
|
@@ -178,6 +206,7 @@ cli
|
|
|
178
206
|
resolvedCommit: commit,
|
|
179
207
|
output: path,
|
|
180
208
|
previous,
|
|
209
|
+
preset,
|
|
181
210
|
});
|
|
182
211
|
};
|
|
183
212
|
if (o.dryRun) {
|
|
@@ -187,18 +216,20 @@ cli
|
|
|
187
216
|
await validateBuild(b);
|
|
188
217
|
} else {
|
|
189
218
|
let source;
|
|
219
|
+
let preset = presetFor(o.embedding);
|
|
190
220
|
if (o.path) source = await identity(o.path, o.remote);
|
|
191
221
|
else {
|
|
192
222
|
invariant(o.repo, "Use --path for a credential-free dry run.");
|
|
193
223
|
const { remote, settings } = await connected();
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
);
|
|
224
|
+
const selected = await selectRepository(remote, o.repo);
|
|
225
|
+
preset = selected.preset ?? PRESET;
|
|
226
|
+
source = await attachment(settings, selected);
|
|
198
227
|
}
|
|
199
228
|
b = await buildLocal(
|
|
200
229
|
source,
|
|
201
230
|
o.previous ? await loadBuild(o.previous) : undefined,
|
|
231
|
+
undefined,
|
|
232
|
+
preset,
|
|
202
233
|
);
|
|
203
234
|
}
|
|
204
235
|
output({
|
|
@@ -239,6 +270,7 @@ cli
|
|
|
239
270
|
await attachment(settings, r),
|
|
240
271
|
undefined,
|
|
241
272
|
pending ? undefined : state,
|
|
273
|
+
r.preset ?? PRESET,
|
|
242
274
|
);
|
|
243
275
|
let baseline: { version: Published; build: Build } | undefined;
|
|
244
276
|
// Tracked branches discover their own immutable baseline remotely.
|
|
@@ -295,6 +327,14 @@ cli
|
|
|
295
327
|
.requiredOption("--version <commit-branch-or-release>")
|
|
296
328
|
.requiredOption("--query <text>")
|
|
297
329
|
.option("--limit <count>", "Maximum hits", integer, 10)
|
|
330
|
+
.addOption(
|
|
331
|
+
new Option(
|
|
332
|
+
"--mode <mode>",
|
|
333
|
+
"Retrieval method; semantic/hybrid incur managed query embedding usage",
|
|
334
|
+
)
|
|
335
|
+
.choices(["lexical", "semantic", "hybrid"])
|
|
336
|
+
.default("lexical"),
|
|
337
|
+
)
|
|
298
338
|
.option("--path <path>", "Exact path filter")
|
|
299
339
|
.option("--language <name>")
|
|
300
340
|
.action(async (o) => {
|
|
@@ -303,10 +343,19 @@ cli
|
|
|
303
343
|
store = remote.store(r.collection),
|
|
304
344
|
v = await resolveVersion(store, r, o.version);
|
|
305
345
|
output(
|
|
306
|
-
await search(
|
|
307
|
-
|
|
308
|
-
|
|
309
|
-
|
|
346
|
+
await search(
|
|
347
|
+
store,
|
|
348
|
+
settings,
|
|
349
|
+
r,
|
|
350
|
+
v,
|
|
351
|
+
o.query,
|
|
352
|
+
o.limit,
|
|
353
|
+
{
|
|
354
|
+
path: o.path,
|
|
355
|
+
language: o.language,
|
|
356
|
+
},
|
|
357
|
+
o.mode,
|
|
358
|
+
),
|
|
310
359
|
);
|
|
311
360
|
});
|
|
312
361
|
cli
|
package/src/publish.ts
CHANGED
|
@@ -10,7 +10,14 @@ import {
|
|
|
10
10
|
sleep,
|
|
11
11
|
type Doc,
|
|
12
12
|
} from "./common.js";
|
|
13
|
-
import {
|
|
13
|
+
import {
|
|
14
|
+
corpusMetadata,
|
|
15
|
+
records,
|
|
16
|
+
recordHash,
|
|
17
|
+
supportedPreset,
|
|
18
|
+
validateBuild,
|
|
19
|
+
type Build,
|
|
20
|
+
} from "./build.js";
|
|
14
21
|
import { branch, tag, one, type CollectionStore } from "./remote.js";
|
|
15
22
|
export type Binding = {
|
|
16
23
|
repoId: string;
|
|
@@ -154,7 +161,11 @@ export async function validateCandidate(
|
|
|
154
161
|
for (const d of corpusMetadata(b, binding.repoId, binding.indexId, attemptId))
|
|
155
162
|
expected.set(d.id, hash(d));
|
|
156
163
|
for await (const d of store.list(tag(name))) {
|
|
157
|
-
|
|
164
|
+
try {
|
|
165
|
+
if (expected.get(d.id) !== recordHash(d, b.preset)) return false;
|
|
166
|
+
} catch {
|
|
167
|
+
return false;
|
|
168
|
+
}
|
|
158
169
|
expected.delete(d.id);
|
|
159
170
|
}
|
|
160
171
|
if (expected.size) return false;
|
|
@@ -203,7 +214,8 @@ export async function validateCandidate(
|
|
|
203
214
|
!found.length ||
|
|
204
215
|
found.some(
|
|
205
216
|
(h) =>
|
|
206
|
-
h.doc.kind !== "chunk" ||
|
|
217
|
+
h.doc.kind !== "chunk" ||
|
|
218
|
+
b.recordHashes[h.doc.id] !== recordHash(h.doc, b.preset),
|
|
207
219
|
)
|
|
208
220
|
)
|
|
209
221
|
return false;
|
|
@@ -284,8 +296,8 @@ export async function publish(args: {
|
|
|
284
296
|
"Build does not belong to this index.",
|
|
285
297
|
);
|
|
286
298
|
invariant(
|
|
287
|
-
!b.preset.embedding,
|
|
288
|
-
"
|
|
299
|
+
!b.preset.embedding || supportedPreset(b.preset),
|
|
300
|
+
"Publication requires a supported lexical or managed embedding preset.",
|
|
289
301
|
);
|
|
290
302
|
await validateBuild(b);
|
|
291
303
|
const journalPath = join(state, "pending.json");
|
|
@@ -522,7 +534,7 @@ export async function publish(args: {
|
|
|
522
534
|
if (j.baseline)
|
|
523
535
|
for await (const d of store.list(tag(j.baseline.tagName))) {
|
|
524
536
|
oldIds.add(d.id);
|
|
525
|
-
oldHashes.set(d.id,
|
|
537
|
+
oldHashes.set(d.id, recordHash(d, b.preset));
|
|
526
538
|
}
|
|
527
539
|
const newIds = new Set([
|
|
528
540
|
...Object.keys(b.recordHashes),
|
package/src/remote.ts
CHANGED
|
@@ -7,7 +7,7 @@ import {
|
|
|
7
7
|
tagTarget,
|
|
8
8
|
type IndexConfigsUnion,
|
|
9
9
|
} from "@functional-systems/lambdadb";
|
|
10
|
-
import { invariant, type Doc } from "./common.js";
|
|
10
|
+
import { hash, invariant, type Doc } from "./common.js";
|
|
11
11
|
import type { Settings } from "./settings.js";
|
|
12
12
|
export type Ref = { kind: "branch" | "tag"; name: string };
|
|
13
13
|
export type Snapshot = { name: string; snapshotId: string };
|
|
@@ -132,7 +132,48 @@ export class LambdaRemote {
|
|
|
132
132
|
pageToken: cursor,
|
|
133
133
|
}),
|
|
134
134
|
);
|
|
135
|
-
|
|
135
|
+
// Some deployments omit managed vectors from list responses even with
|
|
136
|
+
// includeVectors. Fetch only those records from the same immutable Tag;
|
|
137
|
+
// require identical non-vector payloads rather than weakening validation.
|
|
138
|
+
const missing = page.docs
|
|
139
|
+
.map((d) => d.doc as Doc)
|
|
140
|
+
.filter(
|
|
141
|
+
(d) =>
|
|
142
|
+
d.embeddingStatus === "managed" && d.embedding === undefined,
|
|
143
|
+
);
|
|
144
|
+
const hydrated = new Map<string, Doc>();
|
|
145
|
+
if (missing.length) {
|
|
146
|
+
invariant(
|
|
147
|
+
r.kind === "tag",
|
|
148
|
+
"Managed vector hydration requires an immutable Tag.",
|
|
149
|
+
);
|
|
150
|
+
const fetched = await request("fetch listed vectors", () =>
|
|
151
|
+
c.docs.fetch({
|
|
152
|
+
ref: tagRef(r.name),
|
|
153
|
+
ids: missing.map((d) => d.id),
|
|
154
|
+
consistentRead: false,
|
|
155
|
+
includeVectors: true,
|
|
156
|
+
}),
|
|
157
|
+
);
|
|
158
|
+
const expected = new Map(missing.map((d) => [d.id, hash(d)]));
|
|
159
|
+
for (const item of fetched.docs) {
|
|
160
|
+
const doc = item.doc as Doc;
|
|
161
|
+
const { embedding: _, ...source } = doc;
|
|
162
|
+
invariant(
|
|
163
|
+
!hydrated.has(doc.id) && expected.get(doc.id) === hash(source),
|
|
164
|
+
"Listed/fetched managed payloads disagree.",
|
|
165
|
+
);
|
|
166
|
+
hydrated.set(doc.id, doc);
|
|
167
|
+
}
|
|
168
|
+
invariant(
|
|
169
|
+
hydrated.size === expected.size,
|
|
170
|
+
"Listed managed records are missing from fetch.",
|
|
171
|
+
);
|
|
172
|
+
}
|
|
173
|
+
for (const d of page.docs) {
|
|
174
|
+
const doc = d.doc as Doc;
|
|
175
|
+
yield hydrated.get(doc.id) ?? doc;
|
|
176
|
+
}
|
|
136
177
|
cursor = page.nextPageToken || undefined;
|
|
137
178
|
if (cursor) {
|
|
138
179
|
invariant(
|
|
@@ -207,6 +248,7 @@ export class LambdaRemote {
|
|
|
207
248
|
consistentRead: false,
|
|
208
249
|
query,
|
|
209
250
|
size,
|
|
251
|
+
includeVectors: true,
|
|
210
252
|
})
|
|
211
253
|
).docs.map((h) => ({ doc: h.doc as Doc, score: h.score })),
|
|
212
254
|
),
|