@functional-systems/srcx 0.1.0-dev.7 → 0.1.0-dev.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +17 -0
- package/DESIGN.md +55 -21
- package/README.md +70 -1
- package/VALIDATION.md +118 -0
- package/dist/build.d.ts +69 -0
- package/dist/build.js +118 -19
- package/dist/build.js.map +1 -1
- package/dist/cli.js +20 -7
- package/dist/cli.js.map +1 -1
- package/dist/publish.js +11 -5
- package/dist/publish.js.map +1 -1
- package/dist/remote.js +38 -9
- package/dist/remote.js.map +1 -1
- package/dist/repository.d.ts +4 -1
- package/dist/repository.js +15 -11
- package/dist/repository.js.map +1 -1
- package/dist/search.d.ts +7 -2
- package/dist/search.js +24 -4
- package/dist/search.js.map +1 -1
- package/package.json +5 -3
- package/src/build.ts +153 -22
- package/src/cli.ts +59 -10
- package/src/publish.ts +18 -6
- package/src/remote.ts +59 -13
- package/src/repository.ts +20 -11
- package/src/search.ts +37 -8
package/dist/repository.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { randomUUID } from "node:crypto";
|
|
2
2
|
import { join } from "node:path";
|
|
3
3
|
import { atomic, hash, invariant, optionalJson, PURPOSE, } from "./common.js";
|
|
4
|
-
import { PRESET,
|
|
4
|
+
import { PRESET, indexConfigs, supportedPreset } from "./build.js";
|
|
5
5
|
import { identity } from "./git.js";
|
|
6
6
|
import { one, branch } from "./remote.js";
|
|
7
7
|
import { exclusive } from "./publish.js";
|
|
@@ -18,13 +18,13 @@ export function collectionName(key, name, configHash) {
|
|
|
18
18
|
return `code-${slug}-${hash(["collection-v1", key, configHash]).slice(0, 16)}`;
|
|
19
19
|
}
|
|
20
20
|
/** LambdaDB adds its built-in keyword id index to collection metadata. */
|
|
21
|
-
export function matchesIndexSchema(actual) {
|
|
21
|
+
export function matchesIndexSchema(actual, preset = PRESET) {
|
|
22
22
|
if (!actual || typeof actual !== "object" || Array.isArray(actual))
|
|
23
23
|
return false;
|
|
24
24
|
const { id, ...fields } = actual;
|
|
25
25
|
if (id !== undefined && hash(id) !== hash({ type: "keyword" }))
|
|
26
26
|
return false;
|
|
27
|
-
return hash(fields) === hash(
|
|
27
|
+
return hash(fields) === hash(indexConfigs(preset));
|
|
28
28
|
}
|
|
29
29
|
function descriptor(r) {
|
|
30
30
|
return {
|
|
@@ -37,24 +37,25 @@ function descriptor(r) {
|
|
|
37
37
|
repoId: r.repoId,
|
|
38
38
|
indexId: r.indexId,
|
|
39
39
|
name: r.name,
|
|
40
|
-
preset: PRESET,
|
|
40
|
+
preset: r.preset ?? PRESET,
|
|
41
41
|
initialization: "ready",
|
|
42
42
|
};
|
|
43
43
|
}
|
|
44
44
|
function decode(doc, c) {
|
|
45
45
|
invariant(doc?.role === "repository" &&
|
|
46
46
|
doc.initialization === "ready" &&
|
|
47
|
-
doc.
|
|
48
|
-
|
|
47
|
+
supportedPreset(doc.preset) &&
|
|
48
|
+
doc.configHash === hash(doc.preset), "Repository is partially initialized or uses an unsupported preset.");
|
|
49
49
|
invariant(doc.indexId === c.tags["index-id"] &&
|
|
50
50
|
doc.configHash === c.tags["config-hash"] &&
|
|
51
51
|
c.tags.purpose === PURPOSE, "Repository descriptor and Collection labels disagree.");
|
|
52
|
-
invariant(matchesIndexSchema(c.indexConfigs), "Collection schema differs from the pinned preset.");
|
|
52
|
+
invariant(matchesIndexSchema(c.indexConfigs, doc.preset), "Collection schema differs from the pinned preset.");
|
|
53
53
|
invariant(typeof doc.repoKey === "string" &&
|
|
54
54
|
typeof doc.repoId === "string" &&
|
|
55
55
|
typeof doc.indexId === "string" &&
|
|
56
56
|
typeof doc.name === "string", "Malformed repository descriptor.");
|
|
57
57
|
return {
|
|
58
|
+
preset: doc.preset,
|
|
58
59
|
collection: c.collectionName,
|
|
59
60
|
repoKey: doc.repoKey,
|
|
60
61
|
repoId: doc.repoId,
|
|
@@ -91,13 +92,15 @@ export async function selectRepository(remote, selector) {
|
|
|
91
92
|
r.repoKey.split("/").slice(1).join("/") === selector ||
|
|
92
93
|
r.collection === selector);
|
|
93
94
|
invariant(selected.length === 1, selected.length
|
|
94
|
-
? "Ambiguous repository; use its
|
|
95
|
+
? "Ambiguous repository; use its exact Collection name (presets have separate Collections)."
|
|
95
96
|
: `Repository not found${partial.length ? "; partially initialized collections exist (see repo list)" : ""}.`);
|
|
96
97
|
return selected[0];
|
|
97
98
|
}
|
|
98
99
|
export async function register(remote, args) {
|
|
99
100
|
const source = await identity(args.path, args.remote);
|
|
100
|
-
const
|
|
101
|
+
const preset = args.preset ?? PRESET;
|
|
102
|
+
invariant(supportedPreset(preset), "Unsupported repository preset.");
|
|
103
|
+
const configHash = hash(preset), collection = collectionName(source.key, source.name, configHash), state = destination(remote.settings, collection);
|
|
101
104
|
return exclusive(state, async () => {
|
|
102
105
|
const initPath = join(state, "initialization.json");
|
|
103
106
|
let intent = await optionalJson(initPath);
|
|
@@ -127,6 +130,7 @@ export async function register(remote, args) {
|
|
|
127
130
|
invariant(description.length <= 255, "Description exceeds 255 characters.");
|
|
128
131
|
const indexId = randomUUID();
|
|
129
132
|
intent = {
|
|
133
|
+
preset,
|
|
130
134
|
repoKey: source.key,
|
|
131
135
|
name: source.name,
|
|
132
136
|
repoId: randomUUID(),
|
|
@@ -149,9 +153,9 @@ export async function register(remote, args) {
|
|
|
149
153
|
invariant(intent.repoKey === source.key && intent.configHash === configHash, "Provisioning journal identity mismatch.");
|
|
150
154
|
if (existing)
|
|
151
155
|
invariant(hash(existing.tags) === hash(intent.tags) &&
|
|
152
|
-
matchesIndexSchema(existing.indexConfigs), "Existing Collection differs from provisioning journal.");
|
|
156
|
+
matchesIndexSchema(existing.indexConfigs, preset), "Existing Collection differs from provisioning journal.");
|
|
153
157
|
else
|
|
154
|
-
await remote.create(collection, JSON.parse(JSON.stringify(
|
|
158
|
+
await remote.create(collection, JSON.parse(JSON.stringify(indexConfigs(preset))), intent.description, intent.tags);
|
|
155
159
|
const store = remote.store(collection);
|
|
156
160
|
const branches = await store.branches();
|
|
157
161
|
const empty = branches.find((b) => b.name === "checkpoint-empty");
|
package/dist/repository.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"repository.js","sourceRoot":"","sources":["../src/repository.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EACL,MAAM,EACN,IAAI,EACJ,SAAS,EACT,YAAY,EACZ,OAAO,GAER,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,MAAM,EAAE,
|
|
1
|
+
{"version":3,"file":"repository.js","sourceRoot":"","sources":["../src/repository.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EACL,MAAM,EACN,IAAI,EACJ,SAAS,EACT,YAAY,EACZ,OAAO,GAER,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,MAAM,EAAE,YAAY,EAAE,eAAe,EAAe,MAAM,YAAY,CAAC;AAChF,OAAO,EAAE,QAAQ,EAAiB,MAAM,UAAU,CAAC;AACnD,OAAO,EAAgB,GAAG,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACxD,OAAO,EAAE,SAAS,EAAgB,MAAM,cAAc,CAAC;AACvD,OAAO,EAAE,SAAS,EAAiB,MAAM,eAAe,CAAC;AASzD,MAAM,UAAU,WAAW,CAAC,CAAW,EAAE,UAAkB;IACzD,OAAO,IAAI,CACT,SAAS,EAAE,EACX,cAAc,EACd,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,EAC7B,UAAU,CACX,CAAC;AACJ,CAAC;AACD,MAAM,UAAU,cAAc,CAC5B,GAAW,EACX,IAAY,EACZ,UAAkB;IAElB,MAAM,IAAI,GACR,IAAI;SACD,WAAW,EAAE;SACb,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,MAAM,CAAC;IAC5B,OAAO,QAAQ,IAAI,IAAI,IAAI,CAAC,CAAC,eAAe,EAAE,GAAG,EAAE,UAAU,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;AACjF,CAAC;AACD,0EAA0E;AAC1E,MAAM,UAAU,kBAAkB,CAChC,MAAe,EACf,SAAiB,MAAM;IAEvB,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC;QAChE,OAAO,KAAK,CAAC;IACf,MAAM,EAAE,EAAE,EAAE,GAAG,MAAM,EAAE,GAAG,MAAiC,CAAC;IAC5D,IAAI,EAAE,KAAK,SAAS,IAAI,IAAI,CAAC,EAAE,CAAC,KAAK,IAAI,CAAC,EAAE,IAAI,EAAE,SAAS,EAAE,CAAC;QAAE,OAAO,KAAK,CAAC;IAC7E,OAAO,IAAI,CAAC,MAAM,CAAC,KAAK,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAAC;AACrD,CAAC;AACD,SAAS,UAAU,CAAC,CAAa;IAC/B,OAAO;QACL,EAAE,EAAE,UAAU;QACd,IAAI,EAAE,UAAU;QAChB,IAAI,EAAE,YAAY;QAClB,aAAa,EAAE,CAAC;QAChB,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,MAAM,EAAE,CAAC,CAAC,MAAM,IAAI,MAAM;QAC1B,cAAc,EAAE,OAAO;KACxB,CAAC;AACJ,CAAC;AACD,SAAS,MAAM,CACb,GAAoB,EACpB,CAKC;IAED,SAAS,CACP,GAAG,EAAE,IAAI,KAAK,YAAY;QACxB,GAAG,CAAC,cAAc,KAAK,OAAO;QAC9B,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC;QAC3B,GAAG,CAAC,UAAU,KAAK,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,EACrC,oEAAoE,CACrE,CAAC;IACF,SAAS,CACP,GAAG,CAAC,OAAO,KAAK,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC;QAChC,GAAG,CAAC,UAAU,KAAK,CAAC,CAAC,IAAI,CAAC,aAAa,CAAC;QACxC,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,OAAO,EAC5B,uDAAuD,CACxD,CAAC;IACF,SAAS,CACP,kBAAkB,CAAC,CAAC,CAAC,YAAY,EAAE,GAAG,CAAC,MAAgB,CAAC,EACxD,mDAAmD,CACpD,CAAC;IACF,SAAS,CACP,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;QAC7B,OAAO,GAAG,CAAC,MAAM,KAAK,QAAQ;QAC9B,OAAO,GAAG,CAAC,OAAO,KAAK,QAAQ;QAC/B,OAAO,GAAG,CAAC,IAAI,KAAK,QAAQ,EAC9B,kCAAkC,CACnC,CAAC;IACF,OAAO;QACL,MAAM,EAAE,GAAG,CAAC,MAAgB;QAC5B,UAAU,EAAE,CAAC,CAAC,cAAc;QAC5B,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,MAAM,EAAE,GAAG,CAAC,MAAM;QAClB,OAAO,EAAE,GAAG,CAAC,OAAO;QACpB,IAAI,EAAE,GAAG,CAAC,IAAI;QACd,UAAU,EAAE,MAAM,CAAC,GAAG,CAAC,UAAU,CAAC;QAClC,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,IAAI,EAAE,CAAC,CAAC,IAAI;KACb,CAAC;AACJ,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAAC,MAAoB;IAIjD,MAAM,YAAY,GAAiB,EAAE,CAAC;IACtC,MAAM,OAAO,GAA6C,EAAE,CAAC;IAC7D,KAAK,MAAM,CAAC,IAAI,MAAM,MAAM,CAAC,WAAW,EAAE,EAAE,CAAC;QAC3C,IAAI,CAAC,CAAC,IAAI,CAAC,OAAO,KAAK,OAAO;YAAE,SAAS;QACzC,MAAM,GAAG,GAAG,MAAM,GAAG,CACnB,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,cAAc,CAAC,EAC9B,MAAM,CAAC,MAAM,CAAC,EACd,UAAU,EACV,IAAI,CACL,CAAC;QACF,IAAI,CAAC;YACH,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;QACpC,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,CAAC,IAAI,CAAC;gBACX,UAAU,EAAE,CAAC,CAAC,cAAc;gBAC5B,MAAM,EAAE,uDAAuD;aAChE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO,EAAE,YAAY,EAAE,OAAO,EAAE,CAAC;AACnC,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,MAAoB,EACpB,QAAgB;IAEhB,MAAM,EAAE,YAAY,EAAE,OAAO,EAAE,GAAG,MAAM,QAAQ,CAAC,MAAM,CAAC,CAAC;IACzD,MAAM,QAAQ,GAAG,YAAY,CAAC,MAAM,CAClC,CAAC,CAAC,EAAE,EAAE,CACJ,CAAC,CAAC,IAAI,KAAK,QAAQ;QACnB,CAAC,CAAC,OAAO,KAAK,QAAQ;QACtB,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,KAAK,QAAQ;QACpD,CAAC,CAAC,UAAU,KAAK,QAAQ,CAC5B,CAAC;IACF,SAAS,CACP,QAAQ,CAAC,MAAM,KAAK,CAAC,EACrB,QAAQ,CAAC,MAAM;QACb,CAAC,CAAC,0FAA0F;QAC5F,CAAC,CAAC,uBAAuB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,2DAA2D,CAAC,CAAC,CAAC,EAAE,GAAG,CAChH,CAAC;IACF,OAAO,QAAQ,CAAC,CAAC,CAAE,CAAC;AACtB,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,QAAQ,CAC5B,MAAoB,EACpB,IAMC;IAED,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,IAAI,CAAC,MAAM,CAAC,CAAC;IACtD,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,MAAM,CAAC;IACrC,SAAS,CAAC,eAAe,CAAC,MAAM,CAAC,EAAE,gCAAgC,CAAC,CAAC;IACrE,MAAM,UAAU,GAAG,IAAI,CAAC,MAAM,CAAC,EAC7B,UAAU,GAAG,cAAc,CAAC,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,EAAE,UAAU,CAAC,EAChE,KAAK,GAAG,WAAW,CAAC,MAAM,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;IACnD,OAAO,SAAS,CAAC,KAAK,EAAE,KAAK,IAAI,EAAE;QACjC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,EAAE,qBAAqB,CAAC,CAAC;QACpD,IAAI,MAAM,GAAG,MAAM,YAAY,CAAa,QAAQ,CAAC,CAAC;QACtD,MAAM,QAAQ,GAAG,CAAC,MAAM,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CAChD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,UAAU,CACvC,CAAC;QACF,IAAI,QAAQ,EAAE,CAAC;YACb,MAAM,CAAC,GAAG,MAAM,GAAG,CACjB,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,EACxB,MAAM,CAAC,MAAM,CAAC,EACd,UAAU,EACV,IAAI,CACL,CAAC;YACF,IAAI,CAAC,EAAE,CAAC;gBACN,MAAM,MAAM,GAAG,MAAM,CAAC,CAAC,EAAE,QAAQ,CAAC,CAAC;gBACnC,SAAS,CAAC,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,GAAG,EAAE,4BAA4B,CAAC,CAAC;gBACvE,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC,CAAC;gBACrD,OAAO,MAAM,CAAC;YAChB,CAAC;QACH,CAAC;QACD,IAAI,CAAC,MAAM,EAAE,CAAC;YACZ,SAAS,CACP,CAAC,QAAQ,EACT,4FAA4F,CAC7F,CAAC;YACF,MAAM,MAAM,GAAG,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC;YACjC,MAAM,QAAQ,GAAG,CAAC,SAAS,EAAE,YAAY,EAAE,UAAU,EAAE,aAAa,CAAC,CAAC;YACtE,SAAS,CACP,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,MAAM,IAAI,CAAC;gBAC7B,CAAC,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,EACxD,kDAAkD,CACnD,CAAC;YACF,SAAS,CACP,MAAM,CAAC,OAAO,CAAC,MAAM,CAAC,CAAC,KAAK,CAC1B,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,EAAE,CACT,wBAAwB,CAAC,IAAI,CAAC,CAAC,CAAC;gBAChC,CAAC,CAAC,MAAM,IAAI,CAAC;gBACb,CAAC,CAAC,MAAM,IAAI,GAAG;gBACf,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBACnB,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CACnB,EACD,uBAAuB,CACxB,CAAC;YACF,MAAM,WAAW,GACf,IAAI,CAAC,WAAW;gBAChB,wCAAwC,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,8BAA8B,CAAC;YACjG,SAAS,CACP,WAAW,CAAC,MAAM,IAAI,GAAG,EACzB,qCAAqC,CACtC,CAAC;YACF,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;YAC7B,MAAM,GAAG;gBACP,MAAM;gBACN,OAAO,EAAE,MAAM,CAAC,GAAG;gBACnB,IAAI,EAAE,MAAM,CAAC,IAAI;gBACjB,MAAM,EAAE,UAAU,EAAE;gBACpB,OAAO;gBACP,UAAU;gBACV,UAAU;gBACV,WAAW;gBACX,IAAI,EAAE;oBACJ,OAAO,EAAE,OAAO;oBAChB,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,MAAM,IAAI,GAAG,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC;wBACvD,CAAC,CAAC,EAAE,UAAU,EAAE,MAAM,CAAC,GAAG,EAAE;wBAC5B,CAAC,CAAC,EAAE,CAAC;oBACP,UAAU,EAAE,OAAO;oBACnB,aAAa,EAAE,UAAU;oBACzB,GAAG,MAAM;iBACV;aACF,CAAC;YACF,MAAM,MAAM,CAAC,QAAQ,EAAE,MAAM,CAAC,CAAC;QACjC,CAAC;QACD,SAAS,CACP,MAAM,CAAC,OAAO,KAAK,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,UAAU,KAAK,UAAU,EACjE,yCAAyC,CAC1C,CAAC;QACF,IAAI,QAAQ;YACV,SAAS,CACP,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC;gBACvC,kBAAkB,CAAC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC,EACnD,wDAAwD,CACzD,CAAC;;YAEF,MAAM,MAAM,CAAC,MAAM,CACjB,UAAU,EACV,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,YAAY,CAAC,MAAM,CAAC,CAAC,CAG9C,EACD,MAAM,CAAC,WAAW,EAClB,MAAM,CAAC,IAAI,CACZ,CAAC;QACJ,MAAM,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,UAAU,CAAC,CAAC;QACvC,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,QAAQ,EAAE,CAAC;QACxC,MAAM,KAAK,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,kBAAkB,CAAC,CAAC;QAClE,IAAI,KAAK;YACP,SAAS,CACP,KAAK,CAAC,UAAU,KAAK,IAAI,IAAI,KAAK,CAAC,MAAM,KAAK,MAAM,EACpD,qCAAqC,CACtC,CAAC;aACC,CAAC;YACJ,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,MAAM,CAAC,CAAC;YACrD,SAAS,CACP,IAAI,IAAI,IAAI,CAAC,UAAU,KAAK,IAAI,EAChC,yDAAyD,CAC1D,CAAC;YACF,MAAM,KAAK,CAAC,MAAM,CAAC,kBAAkB,EAAE,MAAM,CAAC,CAAC;QACjD,CAAC;QACD,MAAM,KAAK,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QACjD,MAAM,CAAC,GAAG,CAAC,MAAM,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,IAAI,CACzC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,cAAc,KAAK,UAAU,CACvC,CAAC;QACF,SAAS,CAAC,CAAC,EAAE,sDAAsD,CAAC,CAAC;QACrE,MAAM,MAAM,GAAG,MAAM,CACnB,MAAM,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,MAAM,CAAC,EAAE,UAAU,EAAE,IAAI,CAAC,EAClD,CAAC,CACF,CAAC;QACF,MAAM,MAAM,CAAC,IAAI,CAAC,KAAK,EAAE,iBAAiB,CAAC,EAAE,MAAM,CAAC,CAAC;QACrD,OAAO,MAAM,CAAC;IAChB,CAAC,CAAC,CAAC;AACL,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,CAAW,EACX,CAAa;IAEb,MAAM,KAAK,GAAG,MAAM,YAAY,CAC9B,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,EAAE,iBAAiB,CAAC,CACtD,CAAC;IACF,SAAS,CACP,KAAK,EACL,sEAAsE,CACvE,CAAC;IACF,MAAM,OAAO,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC;IACzD,SAAS,CACP,OAAO,CAAC,GAAG,KAAK,CAAC,CAAC,OAAO,EACzB,wDAAwD,CACzD,CAAC;IACF,OAAO,OAAO,CAAC;AACjB,CAAC"}
|
package/dist/search.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ import { type CollectionStore } from "./remote.js";
|
|
|
2
2
|
import { type Settings } from "./settings.js";
|
|
3
3
|
import type { Repository } from "./repository.js";
|
|
4
4
|
import type { Published } from "./publish.js";
|
|
5
|
-
import type
|
|
5
|
+
import { type InventoryItem } from "./build.js";
|
|
6
6
|
export type Handle = {
|
|
7
7
|
id: string;
|
|
8
8
|
endpoint: string;
|
|
@@ -22,10 +22,15 @@ export declare function lexicalQuery(query: string, filters?: {
|
|
|
22
22
|
path?: string;
|
|
23
23
|
language?: string;
|
|
24
24
|
}): Record<string, unknown>;
|
|
25
|
+
export type SearchMode = "lexical" | "semantic" | "hybrid";
|
|
26
|
+
export declare function retrievalQuery(query: string, size: number, filters: {
|
|
27
|
+
path?: string;
|
|
28
|
+
language?: string;
|
|
29
|
+
}, mode: SearchMode): Record<string, unknown>;
|
|
25
30
|
export declare function search(store: CollectionStore, s: Settings, r: Repository, v: Published, query: string, size?: number, filters?: {
|
|
26
31
|
path?: string;
|
|
27
32
|
language?: string;
|
|
28
|
-
}): Promise<unknown[]>;
|
|
33
|
+
}, mode?: SearchMode): Promise<unknown[]>;
|
|
29
34
|
export declare function loadHandle(id: string): Promise<Handle>;
|
|
30
35
|
export declare function directHandle(store: CollectionStore, s: Settings, r: Repository, v: Published, path: string): Promise<Handle>;
|
|
31
36
|
export declare function readHandle(store: CollectionStore, s: Settings, h: Handle, options?: {
|
package/dist/search.js
CHANGED
|
@@ -3,6 +3,7 @@ import { randomUUID } from "node:crypto";
|
|
|
3
3
|
import { atomic, hash, invariant, json, lineAt } from "./common.js";
|
|
4
4
|
import { one, tag } from "./remote.js";
|
|
5
5
|
import { stateRoot } from "./settings.js";
|
|
6
|
+
import { PRESET, recordHash } from "./build.js";
|
|
6
7
|
export async function inventoryAt(store, r, v) {
|
|
7
8
|
const current = (await store.tags()).find((t) => t.name === v.tagName);
|
|
8
9
|
invariant(current?.snapshotId === v.snapshotId, "Pinned Tag was deleted or recreated.");
|
|
@@ -45,12 +46,31 @@ export function lexicalQuery(query, filters = {}) {
|
|
|
45
46
|
],
|
|
46
47
|
};
|
|
47
48
|
}
|
|
48
|
-
export
|
|
49
|
+
export function retrievalQuery(query, size, filters, mode) {
|
|
50
|
+
invariant(["lexical", "semantic", "hybrid"].includes(mode), "Unknown search mode.");
|
|
51
|
+
const lexical = lexicalQuery(query, filters);
|
|
52
|
+
if (mode === "lexical")
|
|
53
|
+
return lexical;
|
|
54
|
+
const semantic = {
|
|
55
|
+
knn: {
|
|
56
|
+
field: "embedding",
|
|
57
|
+
queryText: query,
|
|
58
|
+
k: size,
|
|
59
|
+
filter: {
|
|
60
|
+
bool: lexical.bool.filter((c) => c.occur === "filter"),
|
|
61
|
+
},
|
|
62
|
+
},
|
|
63
|
+
};
|
|
64
|
+
return mode === "semantic" ? semantic : { rrf: [lexical, semantic] };
|
|
65
|
+
}
|
|
66
|
+
export async function search(store, s, r, v, query, size = 10, filters = {}, mode = "lexical") {
|
|
49
67
|
invariant(Number.isInteger(size) && size > 0 && size <= 100, "Limit must be an integer from 1 to 100.");
|
|
68
|
+
invariant(mode === "lexical" || r.preset?.embedding?.managed, "Semantic/hybrid search requires a managed embedding Collection; register with --embedding text-embedding-3-small or text-embedding-3-large.");
|
|
69
|
+
const request = retrievalQuery(query, size, filters, mode);
|
|
50
70
|
const entries = await inventoryAt(store, r, v);
|
|
51
71
|
const files = new Map(entries.filter((e) => e.status === "included").map((e) => [e.fileId, e]));
|
|
52
72
|
const result = [];
|
|
53
|
-
for (const hit of await store.query(v.tagName,
|
|
73
|
+
for (const hit of await store.query(v.tagName, request, size)) {
|
|
54
74
|
const d = hit.doc, e = files.get(d.fileId);
|
|
55
75
|
invariant(d.kind === "chunk" &&
|
|
56
76
|
d.configHash === r.configHash &&
|
|
@@ -68,7 +88,7 @@ export async function search(store, s, r, v, query, size = 10, filters = {}) {
|
|
|
68
88
|
path: String(d.path),
|
|
69
89
|
contentHash: String(d.contentHash),
|
|
70
90
|
chunkId: d.id,
|
|
71
|
-
chunkHash:
|
|
91
|
+
chunkHash: recordHash(d, r.preset ?? PRESET),
|
|
72
92
|
startLine: Number(d.startLine),
|
|
73
93
|
endLine: Number(d.endLine),
|
|
74
94
|
};
|
|
@@ -132,7 +152,7 @@ export async function readHandle(store, s, h, options = {}) {
|
|
|
132
152
|
if (h.chunkId) {
|
|
133
153
|
const d = await one(store, tag(v.tagName), h.chunkId);
|
|
134
154
|
invariant(d &&
|
|
135
|
-
|
|
155
|
+
recordHash(d, h.repository.preset ?? PRESET) === h.chunkHash &&
|
|
136
156
|
d.fileId === h.fileId &&
|
|
137
157
|
d.contentHash === h.contentHash, "Pinned chunk has changed or is missing.");
|
|
138
158
|
const a = Number(d.startByte), z = Number(d.endByte);
|
package/dist/search.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAY,MAAM,aAAa,CAAC;AAC9E,OAAO,EAAE,GAAG,EAAE,GAAG,EAAwB,MAAM,aAAa,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAiB,MAAM,eAAe,CAAC;
|
|
1
|
+
{"version":3,"file":"search.js","sourceRoot":"","sources":["../src/search.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AACjC,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAY,MAAM,aAAa,CAAC;AAC9E,OAAO,EAAE,GAAG,EAAE,GAAG,EAAwB,MAAM,aAAa,CAAC;AAC7D,OAAO,EAAE,SAAS,EAAiB,MAAM,eAAe,CAAC;AAGzD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAsB,MAAM,YAAY,CAAC;AAepE,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,KAAsB,EACtB,CAAa,EACb,CAAY;IAEZ,MAAM,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;IACvE,SAAS,CACP,OAAO,EAAE,UAAU,KAAK,CAAC,CAAC,UAAU,EACpC,sCAAsC,CACvC,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,cAAc,CAAC,CAAC;IAC9D,SAAS,CACP,IAAI,EAAE,MAAM,KAAK,CAAC,CAAC,MAAM;QACvB,IAAI,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO;QAC1B,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;QAChC,IAAI,CAAC,SAAS,KAAK,CAAC,CAAC,SAAS;QAC9B,IAAI,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW;QAClC,IAAI,CAAC,aAAa,KAAK,CAAC,CAAC,aAAa,EACxC,oCAAoC,CACrC,CAAC;IACF,SAAS,CACP,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,gBAAgB,CAAC,EACpC,sCAAsC,CACvC,CAAC;IACF,MAAM,OAAO,GAAoB,EAAE,CAAC;IACpC,KAAK,MAAM,CAAC,CAAC,EAAE,EAAE,CAAC,IAAK,IAAI,CAAC,gBAA6B,CAAC,OAAO,EAAE,EAAE,CAAC;QACpE,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,EAAE,CAAC,CAAC;QAClD,SAAS,CACP,IAAI,EAAE,IAAI,KAAK,WAAW;YACxB,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;YAChC,IAAI,CAAC,WAAW,KAAK,CAAC;YACtB,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,OAAO,CAAC;YAC3B,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,QAAQ,EACtC,yBAAyB,CAC1B,CAAC;QACF,OAAO,CAAC,IAAI,CAAC,GAAI,IAAI,CAAC,OAA2B,CAAC,CAAC;IACrD,CAAC;IACD,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,KAAK,IAAI,CAAC,aAAa,EAAE,0BAA0B,CAAC,CAAC;IAC5E,OAAO,OAAO,CAAC;AACjB,CAAC;AACD,MAAM,UAAU,YAAY,CAC1B,KAAa,EACb,UAAgD,EAAE;IAElD,SAAS,CACP,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,MAAM,IAAI,IAAI,EAC/C,uCAAuC,CACxC,CAAC;IACF,MAAM,OAAO,GAAG,CAAC,KAAa,EAAE,KAAa,EAAE,EAAE,CAAC,CAAC;QACjD,WAAW,EAAE,EAAE,KAAK,EAAE,KAAK,EAAE,YAAY,EAAE,KAAK,EAAE,UAAU,EAAE,IAAI,EAAE;QACpE,KAAK,EAAE,QAAQ;KAChB,CAAC,CAAC;IACH,OAAO;QACL,IAAI,EAAE;YACJ,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC;YACxB;gBACE,WAAW,EAAE,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY,EAAE,UAAU,EAAE,IAAI,EAAE;gBACpE,KAAK,EAAE,MAAM;aACd;YACD,GAAG,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;YACxD,GAAG,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,UAAU,EAAE,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;SACrE;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAC5B,KAAa,EACb,IAAY,EACZ,OAA6C,EAC7C,IAAgB;IAEhB,SAAS,CACP,CAAC,SAAS,EAAE,UAAU,EAAE,QAAQ,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAChD,sBAAsB,CACvB,CAAC;IACF,MAAM,OAAO,GAAG,YAAY,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC;IAC7C,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,OAAO,CAAC;IACvC,MAAM,QAAQ,GAAG;QACf,GAAG,EAAE;YACH,KAAK,EAAE,WAAW;YAClB,SAAS,EAAE,KAAK;YAChB,CAAC,EAAE,IAAI;YACP,MAAM,EAAE;gBACN,IAAI,EAAG,OAAO,CAAC,IAA4B,CAAC,MAAM,CAChD,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,KAAK,QAAQ,CAC5B;aACF;SACF;KACF,CAAC;IACF,OAAO,IAAI,KAAK,UAAU,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,OAAO,EAAE,QAAQ,CAAC,EAAE,CAAC;AACvE,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,KAAsB,EACtB,CAAW,EACX,CAAa,EACb,CAAY,EACZ,KAAa,EACb,IAAI,GAAG,EAAE,EACT,UAAgD,EAAE,EAClD,OAAmB,SAAS;IAE5B,SAAS,CACP,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,IAAI,IAAI,GAAG,EACjD,yCAAyC,CAC1C,CAAC;IACF,SAAS,CACP,IAAI,KAAK,SAAS,IAAI,CAAC,CAAC,MAAM,EAAE,SAAS,EAAE,OAAO,EAClD,6IAA6I,CAC9I,CAAC;IACF,MAAM,OAAO,GAAG,cAAc,CAAC,KAAK,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,CAAC,CAAC;IAC3D,MAAM,OAAO,GAAG,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC;IAC/C,MAAM,KAAK,GAAG,IAAI,GAAG,CACnB,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,KAAK,UAAU,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,CACzE,CAAC;IACF,MAAM,MAAM,GAAG,EAAE,CAAC;IAClB,KAAK,MAAM,GAAG,IAAI,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,EAAE,IAAI,CAAC,EAAE,CAAC;QAC9D,MAAM,CAAC,GAAG,GAAG,CAAC,GAAG,EACf,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,MAAgB,CAAC,CAAC;QACpC,SAAS,CACP,CAAC,CAAC,IAAI,KAAK,OAAO;YAChB,CAAC,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU;YAC7B,CAAC,EAAE,QAAQ,EAAE,QAAQ,CAAC,CAAC,CAAC,EAAE,CAAC;YAC3B,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW;YAC/B,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI,EACnB,qDAAqD,CACtD,CAAC;QACF,MAAM,EAAE,GAAG,UAAU,EAAE,CAAC;QACxB,MAAM,MAAM,GAAW;YACrB,EAAE;YACF,QAAQ,EAAE,CAAC,CAAC,QAAQ;YACpB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,UAAU,EAAE,CAAC;YACb,OAAO,EAAE,CAAC;YACV,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;YACxB,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC;YACpB,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC;YAClC,OAAO,EAAE,CAAC,CAAC,EAAE;YACb,SAAS,EAAE,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,MAAM,IAAI,MAAM,CAAC;YAC5C,SAAS,EAAE,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;YAC9B,OAAO,EAAE,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC;SAC3B,CAAC;QACF,sEAAsE;QACtE,MAAM,QAAQ,GAAG,MAAM,UAAU,CAAC,KAAK,EAAE,CAAC,EAAE,MAAM,CAAC,CAAC;QACpD,MAAM,MAAM,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,EAAE,GAAG,OAAO,CAAC,EAAE,MAAM,CAAC,CAAC;QACjE,MAAM,CAAC,IAAI,CAAC;YACV,QAAQ,EAAE,EAAE;YACZ,UAAU,EAAE,CAAC,CAAC,OAAO;YACrB,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,UAAU,EAAE,CAAC,CAAC,UAAU;YACxB,IAAI,EAAE,CAAC,CAAC,IAAI;YACZ,SAAS,EAAE,CAAC,CAAC,SAAS;YACtB,OAAO,EAAE,CAAC,CAAC,OAAO;YAClB,MAAM,EAAE,CAAC,CAAC,MAAM;YAChB,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,OAAO,EAAE,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC;YAC1C,QAAQ,EAAE,QAAQ,CAAC,QAAQ;SAC5B,CAAC,CAAC;IACL,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,UAAU,CAAC,EAAU;IACzC,SAAS,CAAC,iBAAiB,CAAC,IAAI,CAAC,EAAE,CAAC,EAAE,2BAA2B,CAAC,CAAC;IACnE,MAAM,CAAC,GAAG,MAAM,IAAI,CAAS,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,EAAE,GAAG,OAAO,CAAC,CAAC,CAAC;IACzE,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,EAAE,yBAAyB,CAAC,CAAC;IAClD,OAAO,CAAC,CAAC;AACX,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,YAAY,CAChC,KAAsB,EACtB,CAAW,EACX,CAAa,EACb,CAAY,EACZ,IAAY;IAEZ,MAAM,CAAC,GAAG,CAAC,MAAM,WAAW,CAAC,KAAK,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;IACxE,SAAS,CACP,CAAC,EAAE,MAAM,KAAK,UAAU,IAAI,CAAC,CAAC,MAAM,IAAI,CAAC,CAAC,WAAW,EACrD,sBAAsB,CAAC,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG,CAC1D,CAAC;IACF,OAAO;QACL,EAAE,EAAE,QAAQ;QACZ,QAAQ,EAAE,CAAC,CAAC,QAAQ;QACpB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,UAAU,EAAE,CAAC;QACb,OAAO,EAAE,CAAC;QACV,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,IAAI;QACJ,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,SAAS,EAAE,CAAC;QACZ,OAAO,EAAE,CAAC;KACX,CAAC;AACJ,CAAC;AACD,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,KAAsB,EACtB,CAAW,EACX,CAAS,EACT,UAII,EAAE;IAEN,SAAS,CACP,CAAC,CAAC,QAAQ,KAAK,CAAC,CAAC,QAAQ,IAAI,CAAC,CAAC,OAAO,KAAK,CAAC,CAAC,OAAO,EACpD,qFAAqF,CACtF,CAAC;IACF,MAAM,CAAC,GAAG,CAAC,CAAC,OAAO,EACjB,OAAO,GAAG,CAAC,MAAM,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC;IACnE,SAAS,CACP,OAAO,EAAE,UAAU,KAAK,CAAC,CAAC,UAAU,EACpC,8CAA8C,CAC/C,CAAC;IACF,MAAM,IAAI,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC;IACxD,SAAS,CACP,IAAI,EAAE,IAAI,KAAK,MAAM;QACnB,IAAI,CAAC,UAAU,KAAK,CAAC,CAAC,UAAU,CAAC,UAAU;QAC3C,IAAI,CAAC,IAAI,KAAK,CAAC,CAAC,IAAI;QACpB,OAAO,IAAI,CAAC,UAAU,KAAK,QAAQ;QACnC,IAAI,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW;QAClC,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,KAAK,CAAC,CAAC,WAAW,EACtD,wDAAwD,CACzD,CAAC;IACF,MAAM,GAAG,GAAG,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;IACzC,MAAM,KAAK,GACT,GAAG,CAAC,MAAM,KAAK,CAAC;QACd,CAAC,CAAC,CAAC;QACH,CAAC,CAAC,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAC5D,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;QACd,MAAM,CAAC,GAAG,MAAM,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC;QACtD,SAAS,CACP,CAAC;YACC,UAAU,CAAC,CAAC,EAAE,CAAC,CAAC,UAAU,CAAC,MAAM,IAAI,MAAM,CAAC,KAAK,CAAC,CAAC,SAAS;YAC5D,CAAC,CAAC,MAAM,KAAK,CAAC,CAAC,MAAM;YACrB,CAAC,CAAC,WAAW,KAAK,CAAC,CAAC,WAAW,EACjC,yCAAyC,CAC1C,CAAC;QACF,MAAM,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC,EAC3B,CAAC,GAAG,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;QACxB,SAAS,CACP,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;YACjB,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;YACnB,CAAC,IAAI,CAAC;YACN,CAAC,GAAG,CAAC;YACL,CAAC,IAAI,GAAG,CAAC,MAAM;YACf,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,SAAS;YAC9B,MAAM,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,OAAO,EAClC,uBAAuB,CACxB,CAAC;IACJ,CAAC;IACD,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,CAAC,CAAC;IACrC,SAAS,CACP,MAAM,CAAC,SAAS,CAAC,OAAO,CAAC,IAAI,OAAO,IAAI,CAAC,EACzC,wCAAwC,CACzC,CAAC;IACF,IAAI,KAAK,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,SAAS,EAC3C,GAAG,GAAG,OAAO,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC;IACxC,IAAI,OAAO,CAAC,QAAQ,EAAE,CAAC;QACrB,KAAK,GAAG,CAAC,CAAC;QACV,GAAG,GAAG,KAAK,CAAC;IACd,CAAC;SAAM,CAAC;QACN,SAAS,CACP,MAAM,CAAC,SAAS,CAAC,KAAK,CAAC;YACrB,MAAM,CAAC,SAAS,CAAC,GAAG,CAAC;YACrB,KAAK,IAAI,CAAC;YACV,GAAG,IAAI,KAAK;YACZ,KAAK,IAAI,KAAK;YACd,GAAG,IAAI,KAAK,EACd,iCAAiC,CAClC,CAAC;QACF,KAAK,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,GAAG,OAAO,CAAC,CAAC;QACrC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,GAAG,OAAO,CAAC,CAAC;IACvC,CAAC;IACD,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,mBAAmB,CAAC,IAAI,EAAE,CAAC;IAC/D,MAAM,UAAU,GAAG,OAAO,CAAC,QAAQ;QACjC,CAAC,CAAC,IAAI,CAAC,UAAU;QACjB,CAAC,CAAC,KAAK,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,EAAE,GAAG,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACzC,OAAO;QACL,UAAU,EAAE,CAAC,CAAC,UAAU,CAAC,OAAO;QAChC,SAAS,EAAE,CAAC,CAAC,SAAS;QACtB,OAAO,EAAE,CAAC,CAAC,OAAO;QAClB,UAAU,EAAE,CAAC,CAAC,UAAU;QACxB,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,SAAS,EAAE,KAAK;QAChB,OAAO,EAAE,GAAG;QACZ,WAAW,EAAE,CAAC,CAAC,WAAW;QAC1B,UAAU;QACV,QAAQ,EAAE,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,IAAI,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,IAAI,IAAI,KAAK,IAAI,GAAG,EAAE;KAC7E,CAAC;AACJ,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@functional-systems/srcx",
|
|
3
|
-
"version": "0.1.0-dev.
|
|
3
|
+
"version": "0.1.0-dev.9",
|
|
4
4
|
"private": false,
|
|
5
5
|
"description": "Version-aware code search on LambdaDB",
|
|
6
6
|
"type": "module",
|
|
@@ -37,7 +37,9 @@
|
|
|
37
37
|
"eval:prepare": "npm run build && node scripts/retrieval-eval.mjs prepare",
|
|
38
38
|
"eval:run": "node scripts/retrieval-eval.mjs run",
|
|
39
39
|
"eval:cli:prepare": "npm run build && node scripts/cli-eval.mjs prepare",
|
|
40
|
-
"eval:cli:run": "node scripts/cli-eval.mjs run"
|
|
40
|
+
"eval:cli:run": "node scripts/cli-eval.mjs run",
|
|
41
|
+
"eval:modes:prepare": "npm run build && node scripts/cli-eval.mjs prepare --suite eval/retrieval-modes-v1.json --root .srcx/mode-eval",
|
|
42
|
+
"eval:modes:run": "node scripts/cli-eval.mjs run --root .srcx/mode-eval"
|
|
41
43
|
},
|
|
42
44
|
"dependencies": {
|
|
43
45
|
"@functional-systems/lambdadb": "0.5.1",
|
|
@@ -60,5 +62,5 @@
|
|
|
60
62
|
"url": "https://github.com/lambdadb/srcx/issues"
|
|
61
63
|
},
|
|
62
64
|
"license": "Apache-2.0",
|
|
63
|
-
"gitHead": "
|
|
65
|
+
"gitHead": "4c7b76fc80fd6449095872051ead49f24e77dd5f"
|
|
64
66
|
}
|
package/src/build.ts
CHANGED
|
@@ -30,7 +30,14 @@ export type Preset = {
|
|
|
30
30
|
/** Internal evaluation override; the CLI keeps its original enrichment. */
|
|
31
31
|
enrichment?: "path-only-v1";
|
|
32
32
|
maxFileBytes: number;
|
|
33
|
-
embedding: {
|
|
33
|
+
embedding: {
|
|
34
|
+
model: string;
|
|
35
|
+
dimensions: number;
|
|
36
|
+
managed?: true;
|
|
37
|
+
provider?: "openai";
|
|
38
|
+
similarity?: "cosine";
|
|
39
|
+
sourceField?: "embeddingText";
|
|
40
|
+
} | null;
|
|
34
41
|
};
|
|
35
42
|
export const PRESET: Preset = {
|
|
36
43
|
schemaVersion: 1,
|
|
@@ -39,6 +46,78 @@ export const PRESET: Preset = {
|
|
|
39
46
|
maxFileBytes: 1024 * 1024,
|
|
40
47
|
embedding: null,
|
|
41
48
|
};
|
|
49
|
+
/** Opt-in managed preset; PRESET stays byte-for-byte compatible with lexical corpora. */
|
|
50
|
+
export const MANAGED_PRESET: Preset = {
|
|
51
|
+
...PRESET,
|
|
52
|
+
embedding: {
|
|
53
|
+
managed: true,
|
|
54
|
+
provider: "openai",
|
|
55
|
+
model: "text-embedding-3-small",
|
|
56
|
+
dimensions: 1536,
|
|
57
|
+
similarity: "cosine",
|
|
58
|
+
sourceField: "embeddingText",
|
|
59
|
+
},
|
|
60
|
+
};
|
|
61
|
+
export const MANAGED_LARGE_PRESET: Preset = {
|
|
62
|
+
...MANAGED_PRESET,
|
|
63
|
+
embedding: {
|
|
64
|
+
...MANAGED_PRESET.embedding!,
|
|
65
|
+
model: "text-embedding-3-large",
|
|
66
|
+
dimensions: 3072,
|
|
67
|
+
},
|
|
68
|
+
};
|
|
69
|
+
export function presetFor(embedding = "none"): Preset {
|
|
70
|
+
invariant(
|
|
71
|
+
["none", "text-embedding-3-small", "text-embedding-3-large"].includes(
|
|
72
|
+
embedding,
|
|
73
|
+
),
|
|
74
|
+
"Embedding must be none, text-embedding-3-small or text-embedding-3-large.",
|
|
75
|
+
);
|
|
76
|
+
if (embedding === "none") return PRESET;
|
|
77
|
+
return embedding === "text-embedding-3-large"
|
|
78
|
+
? MANAGED_LARGE_PRESET
|
|
79
|
+
: MANAGED_PRESET;
|
|
80
|
+
}
|
|
81
|
+
export function supportedPreset(preset: unknown): preset is Preset {
|
|
82
|
+
return (
|
|
83
|
+
preset !== undefined &&
|
|
84
|
+
[hash(PRESET), hash(MANAGED_PRESET), hash(MANAGED_LARGE_PRESET)].includes(
|
|
85
|
+
hash(preset),
|
|
86
|
+
)
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
export function indexConfigs(preset: Preset = PRESET) {
|
|
90
|
+
if (!preset.embedding?.managed) return INDEX_CONFIGS;
|
|
91
|
+
const { managed: _, ...embedding } = preset.embedding;
|
|
92
|
+
return {
|
|
93
|
+
...INDEX_CONFIGS,
|
|
94
|
+
embeddingText: { type: "text", analyzers: ["standard"] },
|
|
95
|
+
embedding: { type: "vector", managedEmbedding: true, embedding },
|
|
96
|
+
};
|
|
97
|
+
}
|
|
98
|
+
/** Validate server enrichment separately; all other payload fields remain exact. */
|
|
99
|
+
export function recordHash(doc: Doc, preset: Preset): string {
|
|
100
|
+
if (!preset.embedding?.managed) return hash(doc);
|
|
101
|
+
const { embedding, ...source } = doc;
|
|
102
|
+
if (doc.kind === "chunk" && doc.embeddingStatus === "managed") {
|
|
103
|
+
invariant(
|
|
104
|
+
typeof doc.embeddingText === "string" &&
|
|
105
|
+
doc.embeddingText === doc.searchText &&
|
|
106
|
+
doc.embeddingInputHash ===
|
|
107
|
+
hash([preset.embedding, doc.embeddingText]) &&
|
|
108
|
+
Array.isArray(embedding) &&
|
|
109
|
+
embedding.length === preset.embedding.dimensions &&
|
|
110
|
+
embedding.every((v) => typeof v === "number" && Number.isFinite(v)) &&
|
|
111
|
+
embedding.some((v) => v !== 0),
|
|
112
|
+
"Missing or invalid managed embedding.",
|
|
113
|
+
);
|
|
114
|
+
} else
|
|
115
|
+
invariant(
|
|
116
|
+
embedding === undefined && doc.embeddingText === undefined,
|
|
117
|
+
"Unexpected managed embedding on an ineligible record.",
|
|
118
|
+
);
|
|
119
|
+
return hash(source);
|
|
120
|
+
}
|
|
42
121
|
export type InventoryItem = {
|
|
43
122
|
path: string;
|
|
44
123
|
pathBase64: string;
|
|
@@ -54,6 +133,8 @@ export type InventoryItem = {
|
|
|
54
133
|
parseStatus?: string;
|
|
55
134
|
tokens?: number;
|
|
56
135
|
embedded?: number;
|
|
136
|
+
managed?: number;
|
|
137
|
+
managedTokens?: number;
|
|
57
138
|
skipped?: number;
|
|
58
139
|
};
|
|
59
140
|
export type Build = {
|
|
@@ -152,6 +233,7 @@ export async function materialize(args: {
|
|
|
152
233
|
}
|
|
153
234
|
invariant(
|
|
154
235
|
!preset.embedding ||
|
|
236
|
+
(preset.embedding.managed && supportedPreset(preset)) ||
|
|
155
237
|
(args.embedder?.model === preset.embedding.model &&
|
|
156
238
|
args.embedder.dimensions === preset.embedding.dimensions),
|
|
157
239
|
"Embedding provider must match the pinned preset.",
|
|
@@ -296,31 +378,40 @@ export async function materialize(args: {
|
|
|
296
378
|
const eligible = !["imports", "structural"].includes(span.chunkKind);
|
|
297
379
|
if (preset.embedding && eligible) {
|
|
298
380
|
const inputHash = hash([preset.embedding, span.searchText]);
|
|
299
|
-
|
|
300
|
-
|
|
301
|
-
|
|
302
|
-
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
|
|
306
|
-
|
|
381
|
+
if (preset.embedding.managed) {
|
|
382
|
+
doc.embeddingText = span.searchText;
|
|
383
|
+
doc.embeddingInputHash = inputHash;
|
|
384
|
+
doc.embeddingStatus = "managed";
|
|
385
|
+
delete doc.embeddingSkipReason;
|
|
386
|
+
item.managed = (item.managed ?? 0) + 1;
|
|
387
|
+
item.managedTokens = (item.managedTokens ?? 0) + span.tokenCount;
|
|
388
|
+
} else {
|
|
389
|
+
const cachePath = args.cache
|
|
390
|
+
? join(args.cache, inputHash + ".json")
|
|
391
|
+
: undefined;
|
|
392
|
+
let vector = cachePath
|
|
393
|
+
? await optionalJson<number[]>(cachePath)
|
|
394
|
+
: undefined;
|
|
395
|
+
if (!vector) {
|
|
396
|
+
vector = await args.embedder!.embed(span.searchText);
|
|
397
|
+
invariant(
|
|
398
|
+
vector.length === preset.embedding.dimensions &&
|
|
399
|
+
vector.every(Number.isFinite),
|
|
400
|
+
"Invalid embedding output.",
|
|
401
|
+
);
|
|
402
|
+
if (cachePath) await atomic(cachePath, vector);
|
|
403
|
+
}
|
|
307
404
|
invariant(
|
|
308
405
|
vector.length === preset.embedding.dimensions &&
|
|
309
406
|
vector.every(Number.isFinite),
|
|
310
|
-
"Invalid
|
|
407
|
+
"Invalid cached vector.",
|
|
311
408
|
);
|
|
312
|
-
|
|
409
|
+
doc.embedding = vector;
|
|
410
|
+
doc.embeddingInputHash = inputHash;
|
|
411
|
+
doc.embeddingStatus = "embedded";
|
|
412
|
+
delete doc.embeddingSkipReason;
|
|
413
|
+
item.embedded!++;
|
|
313
414
|
}
|
|
314
|
-
invariant(
|
|
315
|
-
vector.length === preset.embedding.dimensions &&
|
|
316
|
-
vector.every(Number.isFinite),
|
|
317
|
-
"Invalid cached vector.",
|
|
318
|
-
);
|
|
319
|
-
doc.embedding = vector;
|
|
320
|
-
doc.embeddingInputHash = inputHash;
|
|
321
|
-
doc.embeddingStatus = "embedded";
|
|
322
|
-
delete doc.embeddingSkipReason;
|
|
323
|
-
item.embedded!++;
|
|
324
415
|
} else {
|
|
325
416
|
doc.embeddingSkipReason = preset.embedding
|
|
326
417
|
? span.chunkKind === "imports"
|
|
@@ -399,6 +490,15 @@ export async function materialize(args: {
|
|
|
399
490
|
excluded: items.filter((e) => e.status === "excluded").length,
|
|
400
491
|
chunks: items.reduce((n, e) => n + (e.chunkIds?.length ?? 0), 0),
|
|
401
492
|
tokens: items.reduce((n, e) => n + (e.tokens ?? 0), 0),
|
|
493
|
+
...(preset.embedding?.managed
|
|
494
|
+
? {
|
|
495
|
+
managed: items.reduce((n, e) => n + (e.managed ?? 0), 0),
|
|
496
|
+
managedTokens: items.reduce(
|
|
497
|
+
(n, e) => n + (e.managedTokens ?? 0),
|
|
498
|
+
0,
|
|
499
|
+
),
|
|
500
|
+
}
|
|
501
|
+
: {}),
|
|
402
502
|
embedded: items.reduce((n, e) => n + (e.embedded ?? 0), 0),
|
|
403
503
|
skipped: items.reduce((n, e) => n + (e.skipped ?? 0), 0),
|
|
404
504
|
},
|
|
@@ -450,6 +550,17 @@ export async function validateBuild(b: Build): Promise<void> {
|
|
|
450
550
|
doc.configHash === b.configHash && doc.schemaVersion === 1,
|
|
451
551
|
"Mixed configuration.",
|
|
452
552
|
);
|
|
553
|
+
if (b.preset.embedding?.managed) {
|
|
554
|
+
invariant(
|
|
555
|
+
doc.embedding === undefined,
|
|
556
|
+
"Managed artifacts must not contain vectors.",
|
|
557
|
+
);
|
|
558
|
+
if (doc.kind !== "chunk")
|
|
559
|
+
invariant(
|
|
560
|
+
doc.embeddingText === undefined,
|
|
561
|
+
"Only eligible chunks can contain embedding input.",
|
|
562
|
+
);
|
|
563
|
+
}
|
|
453
564
|
if (doc.kind === "file") {
|
|
454
565
|
finish();
|
|
455
566
|
activeFile = doc;
|
|
@@ -505,7 +616,27 @@ export async function validateBuild(b: Build): Promise<void> {
|
|
|
505
616
|
Number(doc.tokenCount) <= CHUNKER.maxTokens,
|
|
506
617
|
"Invalid chunk source/position/token binding.",
|
|
507
618
|
);
|
|
508
|
-
if (
|
|
619
|
+
if (b.preset.embedding?.managed) {
|
|
620
|
+
const eligible = !["imports", "structural"].includes(
|
|
621
|
+
String(doc.chunkKind),
|
|
622
|
+
);
|
|
623
|
+
invariant(
|
|
624
|
+
eligible
|
|
625
|
+
? doc.embeddingStatus === "managed" &&
|
|
626
|
+
doc.embeddingText === doc.searchText &&
|
|
627
|
+
doc.embeddingInputHash ===
|
|
628
|
+
hash([b.preset.embedding, doc.searchText]) &&
|
|
629
|
+
doc.embeddingSkipReason === undefined
|
|
630
|
+
: doc.embeddingStatus === "skipped" &&
|
|
631
|
+
doc.embeddingText === undefined &&
|
|
632
|
+
doc.embeddingInputHash === undefined &&
|
|
633
|
+
doc.embeddingSkipReason ===
|
|
634
|
+
(doc.chunkKind === "imports"
|
|
635
|
+
? "imports-only"
|
|
636
|
+
: "structural-only"),
|
|
637
|
+
"Invalid managed embedding input or eligibility.",
|
|
638
|
+
);
|
|
639
|
+
} else if (doc.embeddingStatus === "embedded")
|
|
509
640
|
invariant(
|
|
510
641
|
b.preset.embedding &&
|
|
511
642
|
Array.isArray(doc.embedding) &&
|