@maadb/core 0.7.17 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +21 -6
- package/dist/backend/adapter.d.ts +7 -1
- package/dist/backend/adapter.d.ts.map +1 -1
- package/dist/backend/sqlite/index.d.ts +9 -1
- package/dist/backend/sqlite/index.d.ts.map +1 -1
- package/dist/backend/sqlite/index.js +34 -2
- package/dist/backend/sqlite/index.js.map +1 -1
- package/dist/backend/sqlite/semantic-store.d.ts +51 -0
- package/dist/backend/sqlite/semantic-store.d.ts.map +1 -0
- package/dist/backend/sqlite/semantic-store.js +284 -0
- package/dist/backend/sqlite/semantic-store.js.map +1 -0
- package/dist/cli/commands/maintain.d.ts.map +1 -1
- package/dist/cli/commands/maintain.js +3 -2
- package/dist/cli/commands/maintain.js.map +1 -1
- package/dist/cli/commands/read.d.ts +1 -0
- package/dist/cli/commands/read.d.ts.map +1 -1
- package/dist/cli/commands/read.js +52 -0
- package/dist/cli/commands/read.js.map +1 -1
- package/dist/cli/index.js +6 -2
- package/dist/cli/index.js.map +1 -1
- package/dist/engine/context.d.ts +12 -0
- package/dist/engine/context.d.ts.map +1 -1
- package/dist/engine/context.js.map +1 -1
- package/dist/engine/index.d.ts +40 -0
- package/dist/engine/index.d.ts.map +1 -1
- package/dist/engine/index.js +115 -9
- package/dist/engine/index.js.map +1 -1
- package/dist/engine/indexing.d.ts.map +1 -1
- package/dist/engine/indexing.js +22 -2
- package/dist/engine/indexing.js.map +1 -1
- package/dist/engine/semantic/config.d.ts +23 -0
- package/dist/engine/semantic/config.d.ts.map +1 -0
- package/dist/engine/semantic/config.js +41 -0
- package/dist/engine/semantic/config.js.map +1 -0
- package/dist/engine/semantic/indexer.d.ts +21 -0
- package/dist/engine/semantic/indexer.d.ts.map +1 -0
- package/dist/engine/semantic/indexer.js +91 -0
- package/dist/engine/semantic/indexer.js.map +1 -0
- package/dist/engine/semantic/provider.d.ts +10 -0
- package/dist/engine/semantic/provider.d.ts.map +1 -0
- package/dist/engine/semantic/provider.js +52 -0
- package/dist/engine/semantic/provider.js.map +1 -0
- package/dist/engine/semantic/providers/fake.d.ts +9 -0
- package/dist/engine/semantic/providers/fake.d.ts.map +1 -0
- package/dist/engine/semantic/providers/fake.js +54 -0
- package/dist/engine/semantic/providers/fake.js.map +1 -0
- package/dist/engine/semantic/providers/openai.d.ts +23 -0
- package/dist/engine/semantic/providers/openai.d.ts.map +1 -0
- package/dist/engine/semantic/providers/openai.js +103 -0
- package/dist/engine/semantic/providers/openai.js.map +1 -0
- package/dist/engine/semantic/rrf.d.ts +9 -0
- package/dist/engine/semantic/rrf.d.ts.map +1 -0
- package/dist/engine/semantic/rrf.js +29 -0
- package/dist/engine/semantic/rrf.js.map +1 -0
- package/dist/engine/semantic/search.d.ts +6 -0
- package/dist/engine/semantic/search.d.ts.map +1 -0
- package/dist/engine/semantic/search.js +202 -0
- package/dist/engine/semantic/search.js.map +1 -0
- package/dist/engine/semantic/types.d.ts +138 -0
- package/dist/engine/semantic/types.d.ts.map +1 -0
- package/dist/engine/semantic/types.js +7 -0
- package/dist/engine/semantic/types.js.map +1 -0
- package/dist/engine/writes.d.ts.map +1 -1
- package/dist/engine/writes.js +5 -0
- package/dist/engine/writes.js.map +1 -1
- package/dist/errors.d.ts +1 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js.map +1 -1
- package/dist/mcp/heavy-ops.d.ts +93 -0
- package/dist/mcp/heavy-ops.d.ts.map +1 -0
- package/dist/mcp/heavy-ops.js +254 -0
- package/dist/mcp/heavy-ops.js.map +1 -0
- package/dist/mcp/kinds.d.ts.map +1 -1
- package/dist/mcp/kinds.js +1 -0
- package/dist/mcp/kinds.js.map +1 -1
- package/dist/mcp/memory-pressure.d.ts +9 -0
- package/dist/mcp/memory-pressure.d.ts.map +1 -1
- package/dist/mcp/memory-pressure.js +16 -0
- package/dist/mcp/memory-pressure.js.map +1 -1
- package/dist/mcp/roles.js +1 -1
- package/dist/mcp/roles.js.map +1 -1
- package/dist/mcp/server.d.ts.map +1 -1
- package/dist/mcp/server.js +5 -0
- package/dist/mcp/server.js.map +1 -1
- package/dist/mcp/tools/maintain.d.ts.map +1 -1
- package/dist/mcp/tools/maintain.js +9 -4
- package/dist/mcp/tools/maintain.js.map +1 -1
- package/dist/mcp/tools/read.d.ts.map +1 -1
- package/dist/mcp/tools/read.js +35 -1
- package/dist/mcp/tools/read.js.map +1 -1
- package/dist/mcp/with-session.d.ts.map +1 -1
- package/dist/mcp/with-session.js +40 -1
- package/dist/mcp/with-session.js.map +1 -1
- package/dist/parser/index.d.ts +14 -1
- package/dist/parser/index.d.ts.map +1 -1
- package/dist/parser/index.js +18 -0
- package/dist/parser/index.js.map +1 -1
- package/dist/types.d.ts +7 -0
- package/dist/types.d.ts.map +1 -1
- package/package.json +4 -1
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import type { EmbeddingProvider, SemanticIndex } from './types.js';
|
|
2
|
+
export declare class SemanticIndexer {
|
|
3
|
+
private readonly index;
|
|
4
|
+
private readonly provider;
|
|
5
|
+
private readonly batchSize;
|
|
6
|
+
private currentDrain;
|
|
7
|
+
private rerun;
|
|
8
|
+
private stopped;
|
|
9
|
+
constructor(index: SemanticIndex, provider: EmbeddingProvider, batchSize?: number);
|
|
10
|
+
/** Crash-resume: drain whatever a prior process left pending. Fire-and-forget. */
|
|
11
|
+
start(): void;
|
|
12
|
+
/** Signal that new work may be queued. Non-blocking. */
|
|
13
|
+
kick(): void;
|
|
14
|
+
/** Awaitable drain to completion (reindex --embeddings, tests). */
|
|
15
|
+
flush(): Promise<void>;
|
|
16
|
+
/** Stop accepting work; lets an in-flight drain finish. */
|
|
17
|
+
stop(): Promise<void>;
|
|
18
|
+
private ensureDraining;
|
|
19
|
+
private drain;
|
|
20
|
+
}
|
|
21
|
+
//# sourceMappingURL=indexer.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"indexer.d.ts","sourceRoot":"","sources":["../../../src/engine/semantic/indexer.ts"],"names":[],"mappings":"AAgBA,OAAO,KAAK,EAAE,iBAAiB,EAAE,aAAa,EAAkB,MAAM,YAAY,CAAC;AAInF,qBAAa,eAAe;IAC1B,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAgB;IACtC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAoB;IAC7C,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAS;IACnC,OAAO,CAAC,YAAY,CAA8B;IAClD,OAAO,CAAC,KAAK,CAAS;IACtB,OAAO,CAAC,OAAO,CAAS;gBAEZ,KAAK,EAAE,aAAa,EAAE,QAAQ,EAAE,iBAAiB,EAAE,SAAS,SAAgB;IAMxF,kFAAkF;IAClF,KAAK,IAAI,IAAI;IAIb,wDAAwD;IACxD,IAAI,IAAI,IAAI;IAKZ,mEAAmE;IAC7D,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;IAK5B,2DAA2D;IACrD,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC;IAO3B,OAAO,CAAC,cAAc;YASR,KAAK;CA0BpB"}
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// Semantic indexer (0.8.0) — the async embed worker. Drains the persisted
|
|
3
|
+
// embed_queue: pull a batch of pending blocks, embed them (kind='document'),
|
|
4
|
+
// upsert vectors, clear the queue rows. Runs OUTSIDE the engine write mutex —
|
|
5
|
+
// model/network latency must never block the write FIFO. The DB writes it does
|
|
6
|
+
// (putBlockEmbeddings) are fast and synchronous; only the embed() call awaits.
|
|
7
|
+
//
|
|
8
|
+
// Lifecycle: start() does a crash-resume drain of whatever the queue holds from
|
|
9
|
+
// a prior process; kick() is fired (fire-and-forget) after each durable write;
|
|
10
|
+
// flush() is the awaitable form used by reindex --embeddings and tests.
|
|
11
|
+
// Single-flight: one drain at a time; a kick during a drain schedules a rerun.
|
|
12
|
+
// Fail-soft: an embed error tallies failures and leaves the queue rows for a
|
|
13
|
+
// later retry rather than dropping them or hot-looping.
|
|
14
|
+
// ============================================================================
|
|
15
|
+
import { logger } from '../logger.js';
|
|
16
|
+
const DEFAULT_BATCH = 32;
|
|
17
|
+
export class SemanticIndexer {
|
|
18
|
+
index;
|
|
19
|
+
provider;
|
|
20
|
+
batchSize;
|
|
21
|
+
currentDrain = null;
|
|
22
|
+
rerun = false;
|
|
23
|
+
stopped = false;
|
|
24
|
+
constructor(index, provider, batchSize = DEFAULT_BATCH) {
|
|
25
|
+
this.index = index;
|
|
26
|
+
this.provider = provider;
|
|
27
|
+
this.batchSize = batchSize > 0 ? batchSize : DEFAULT_BATCH;
|
|
28
|
+
}
|
|
29
|
+
/** Crash-resume: drain whatever a prior process left pending. Fire-and-forget. */
|
|
30
|
+
start() {
|
|
31
|
+
this.kick();
|
|
32
|
+
}
|
|
33
|
+
/** Signal that new work may be queued. Non-blocking. */
|
|
34
|
+
kick() {
|
|
35
|
+
if (this.stopped)
|
|
36
|
+
return;
|
|
37
|
+
void this.ensureDraining();
|
|
38
|
+
}
|
|
39
|
+
/** Awaitable drain to completion (reindex --embeddings, tests). */
|
|
40
|
+
async flush() {
|
|
41
|
+
if (this.stopped)
|
|
42
|
+
return;
|
|
43
|
+
await this.ensureDraining();
|
|
44
|
+
}
|
|
45
|
+
/** Stop accepting work; lets an in-flight drain finish. */
|
|
46
|
+
async stop() {
|
|
47
|
+
this.stopped = true;
|
|
48
|
+
if (this.currentDrain) {
|
|
49
|
+
try {
|
|
50
|
+
await this.currentDrain;
|
|
51
|
+
}
|
|
52
|
+
catch { /* already logged */ }
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
ensureDraining() {
|
|
56
|
+
if (this.currentDrain) {
|
|
57
|
+
this.rerun = true; // fold this request into the running drain
|
|
58
|
+
return this.currentDrain;
|
|
59
|
+
}
|
|
60
|
+
this.currentDrain = this.drain().finally(() => { this.currentDrain = null; });
|
|
61
|
+
return this.currentDrain;
|
|
62
|
+
}
|
|
63
|
+
async drain() {
|
|
64
|
+
do {
|
|
65
|
+
this.rerun = false;
|
|
66
|
+
while (!this.stopped) {
|
|
67
|
+
const batch = this.index.takeEmbedBatch(this.batchSize);
|
|
68
|
+
if (batch.length === 0)
|
|
69
|
+
break;
|
|
70
|
+
try {
|
|
71
|
+
const vectors = await this.provider.embed(batch.map(b => b.text), 'document');
|
|
72
|
+
if (vectors.length !== batch.length) {
|
|
73
|
+
throw new Error(`provider returned ${vectors.length} vectors for ${batch.length} inputs`);
|
|
74
|
+
}
|
|
75
|
+
const rows = batch.map((b, i) => ({
|
|
76
|
+
qid: b.qid, docId: b.docId, blockOrd: b.blockOrd, vector: vectors[i],
|
|
77
|
+
}));
|
|
78
|
+
this.index.putBlockEmbeddings(rows);
|
|
79
|
+
}
|
|
80
|
+
catch (e) {
|
|
81
|
+
// Leave the queue rows in place for a later retry; break to avoid a
|
|
82
|
+
// hot loop on a persistent failure (next kick/flush retries).
|
|
83
|
+
this.index.recordFailure(batch.length);
|
|
84
|
+
logger.bestEffort('engine', 'embed_batch_failed', `embedding batch failed (${batch.length} blocks); left queued for retry: ${e.message}`);
|
|
85
|
+
return;
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
} while (this.rerun && !this.stopped);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=indexer.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"indexer.js","sourceRoot":"","sources":["../../../src/engine/semantic/indexer.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,0EAA0E;AAC1E,6EAA6E;AAC7E,8EAA8E;AAC9E,+EAA+E;AAC/E,+EAA+E;AAC/E,EAAE;AACF,gFAAgF;AAChF,+EAA+E;AAC/E,wEAAwE;AACxE,+EAA+E;AAC/E,6EAA6E;AAC7E,wDAAwD;AACxD,+EAA+E;AAE/E,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAGtC,MAAM,aAAa,GAAG,EAAE,CAAC;AAEzB,MAAM,OAAO,eAAe;IACT,KAAK,CAAgB;IACrB,QAAQ,CAAoB;IAC5B,SAAS,CAAS;IAC3B,YAAY,GAAyB,IAAI,CAAC;IAC1C,KAAK,GAAG,KAAK,CAAC;IACd,OAAO,GAAG,KAAK,CAAC;IAExB,YAAY,KAAoB,EAAE,QAA2B,EAAE,SAAS,GAAG,aAAa;QACtF,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;QACnB,IAAI,CAAC,QAAQ,GAAG,QAAQ,CAAC;QACzB,IAAI,CAAC,SAAS,GAAG,SAAS,GAAG,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,aAAa,CAAC;IAC7D,CAAC;IAED,kFAAkF;IAClF,KAAK;QACH,IAAI,CAAC,IAAI,EAAE,CAAC;IACd,CAAC;IAED,wDAAwD;IACxD,IAAI;QACF,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,KAAK,IAAI,CAAC,cAAc,EAAE,CAAC;IAC7B,CAAC;IAED,mEAAmE;IACnE,KAAK,CAAC,KAAK;QACT,IAAI,IAAI,CAAC,OAAO;YAAE,OAAO;QACzB,MAAM,IAAI,CAAC,cAAc,EAAE,CAAC;IAC9B,CAAC;IAED,2DAA2D;IAC3D,KAAK,CAAC,IAAI;QACR,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC;QACpB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,IAAI,CAAC;gBAAC,MAAM,IAAI,CAAC,YAAY,CAAC;YAAC,CAAC;YAAC,MAAM,CAAC,CAAC,oBAAoB,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAEO,cAAc;QACpB,IAAI,IAAI,CAAC,YAAY,EAAE,CAAC;YACtB,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,CAAY,2CAA2C;YACzE,OAAO,IAAI,CAAC,YAAY,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC,OAAO,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;QAC9E,OAAO,IAAI,CAAC,YAAY,CAAC;IAC3B,CAAC;IAEO,KAAK,CAAC,KAAK;QACjB,GAAG,CAAC;YACF,IAAI,CAAC,KAAK,GAAG,KAAK,CAAC;YACnB,OAAO,CAAC,IAAI,CAAC,OAAO,EAAE,CAAC;gBACrB,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,cAAc,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBACxD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;oBAAE,MAAM;gBAC9B,IAAI,CAAC;oBACH,MAAM,OAAO,GAAG,MAAM,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,UAAU,CAAC,CAAC;oBAC9E,IAAI,OAAO,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;wBACpC,MAAM,IAAI,KAAK,CAAC,qBAAqB,OAAO,CAAC,MAAM,gBAAgB,KAAK,CAAC,MAAM,SAAS,CAAC,CAAC;oBAC5F,CAAC;oBACD,MAAM,IAAI,GAAqB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;wBAClD,GAAG,EAAE,CAAC,CAAC,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC,CAAE;qBACtE,CAAC,CAAC,CAAC;oBACJ,IAAI,CAAC,KAAK,CAAC,kBAAkB,CAAC,IAAI,CAAC,CAAC;gBACtC,CAAC;gBAAC,OAAO,CAAC,EAAE,CAAC;oBACX,oEAAoE;oBACpE,8DAA8D;oBAC9D,IAAI,CAAC,KAAK,CAAC,aAAa,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;oBACvC,MAAM,CAAC,UAAU,CAAC,QAAQ,EAAE,oBAAoB,EAC9C,2BAA2B,KAAK,CAAC,MAAM,oCAAqC,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;oBACrG,OAAO;gBACT,CAAC;YACH,CAAC;QACH,CAAC,QAAQ,IAAI,CAAC,KAAK,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE;IACxC,CAAC;CACF"}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
import type { EmbeddingProvider } from './types.js';
|
|
2
|
+
import { type SemanticEnv } from './config.js';
|
|
3
|
+
export interface ProviderResolveOptions {
|
|
4
|
+
/** Host-injected provider (DI). Wins over env when present. */
|
|
5
|
+
injected?: EmbeddingProvider | undefined;
|
|
6
|
+
/** Override env (tests); defaults to readSemanticEnv(). */
|
|
7
|
+
env?: SemanticEnv | undefined;
|
|
8
|
+
}
|
|
9
|
+
export declare function resolveEmbeddingProvider(opts?: ProviderResolveOptions): EmbeddingProvider | null;
|
|
10
|
+
//# sourceMappingURL=provider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"provider.d.ts","sourceRoot":"","sources":["../../../src/engine/semantic/provider.ts"],"names":[],"mappings":"AAYA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AACpD,OAAO,EAAmB,KAAK,WAAW,EAAE,MAAM,aAAa,CAAC;AAOhE,MAAM,WAAW,sBAAsB;IACrC,+DAA+D;IAC/D,QAAQ,CAAC,EAAE,iBAAiB,GAAG,SAAS,CAAC;IACzC,2DAA2D;IAC3D,GAAG,CAAC,EAAE,WAAW,GAAG,SAAS,CAAC;CAC/B;AAED,wBAAgB,wBAAwB,CAAC,IAAI,GAAE,sBAA2B,GAAG,iBAAiB,GAAG,IAAI,CAoCpG"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// Embedding provider resolution (0.8.0)
|
|
3
|
+
// Two construction paths, in priority order:
|
|
4
|
+
// 1. Injected — the host app passes an EmbeddingProvider; the engine holds no
|
|
5
|
+
// API keys. This is how an embedding host reuses its existing client.
|
|
6
|
+
// 2. Env-constructed — for standalone `maad serve`: MAAD_EMBED_PROVIDER picks
|
|
7
|
+
// openai | fake | local, sized via MAAD_EMBED_MODEL / MAAD_EMBED_DIM.
|
|
8
|
+
// Returns null when no vector provider is configured — semantic stays in
|
|
9
|
+
// lexical-only mode (exact/FTS work; semantic/hybrid degrade to lexical).
|
|
10
|
+
// ============================================================================
|
|
11
|
+
import { logger } from '../logger.js';
|
|
12
|
+
import { readSemanticEnv } from './config.js';
|
|
13
|
+
import { OpenAiEmbeddingProvider } from './providers/openai.js';
|
|
14
|
+
import { FakeEmbeddingProvider } from './providers/fake.js';
|
|
15
|
+
const DEFAULT_OPENAI_MODEL = 'text-embedding-3-small';
|
|
16
|
+
const DEFAULT_FAKE_DIM = 64;
|
|
17
|
+
export function resolveEmbeddingProvider(opts = {}) {
|
|
18
|
+
if (opts.injected)
|
|
19
|
+
return opts.injected;
|
|
20
|
+
const env = opts.env ?? readSemanticEnv();
|
|
21
|
+
switch (env.provider) {
|
|
22
|
+
case 'openai': {
|
|
23
|
+
if (env.openaiApiKey === undefined) {
|
|
24
|
+
logger.degraded('engine', 'semantic_provider_unconfigured', 'MAAD_EMBED_PROVIDER=openai but MAAD_OPENAI_API_KEY is unset; running lexical-only (no vectors)');
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
try {
|
|
28
|
+
return new OpenAiEmbeddingProvider({
|
|
29
|
+
apiKey: env.openaiApiKey,
|
|
30
|
+
model: env.model ?? DEFAULT_OPENAI_MODEL,
|
|
31
|
+
dim: env.dim,
|
|
32
|
+
baseUrl: env.openaiBaseUrl,
|
|
33
|
+
});
|
|
34
|
+
}
|
|
35
|
+
catch (e) {
|
|
36
|
+
logger.degraded('engine', 'semantic_provider_error', `failed to construct OpenAI embedding provider; lexical-only: ${e.message}`);
|
|
37
|
+
return null;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
case 'fake':
|
|
41
|
+
return new FakeEmbeddingProvider(env.dim ?? DEFAULT_FAKE_DIM);
|
|
42
|
+
case 'local':
|
|
43
|
+
// Phase 2 — local transformers.js provider (optional dep). Not in this build.
|
|
44
|
+
logger.degraded('engine', 'semantic_provider_unavailable', 'MAAD_EMBED_PROVIDER=local is not available in this build; ' +
|
|
45
|
+
'inject a provider or use MAAD_EMBED_PROVIDER=openai. Running lexical-only.');
|
|
46
|
+
return null;
|
|
47
|
+
case 'none':
|
|
48
|
+
default:
|
|
49
|
+
return null;
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=provider.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"provider.js","sourceRoot":"","sources":["../../../src/engine/semantic/provider.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,wCAAwC;AACxC,6CAA6C;AAC7C,gFAAgF;AAChF,2EAA2E;AAC3E,gFAAgF;AAChF,2EAA2E;AAC3E,yEAAyE;AACzE,0EAA0E;AAC1E,+EAA+E;AAE/E,OAAO,EAAE,MAAM,EAAE,MAAM,cAAc,CAAC;AAEtC,OAAO,EAAE,eAAe,EAAoB,MAAM,aAAa,CAAC;AAChE,OAAO,EAAE,uBAAuB,EAAE,MAAM,uBAAuB,CAAC;AAChE,OAAO,EAAE,qBAAqB,EAAE,MAAM,qBAAqB,CAAC;AAE5D,MAAM,oBAAoB,GAAG,wBAAwB,CAAC;AACtD,MAAM,gBAAgB,GAAG,EAAE,CAAC;AAS5B,MAAM,UAAU,wBAAwB,CAAC,OAA+B,EAAE;IACxE,IAAI,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC,QAAQ,CAAC;IACxC,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,eAAe,EAAE,CAAC;IAE1C,QAAQ,GAAG,CAAC,QAAQ,EAAE,CAAC;QACrB,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;gBACnC,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,gCAAgC,EACxD,gGAAgG,CAAC,CAAC;gBACpG,OAAO,IAAI,CAAC;YACd,CAAC;YACD,IAAI,CAAC;gBACH,OAAO,IAAI,uBAAuB,CAAC;oBACjC,MAAM,EAAE,GAAG,CAAC,YAAY;oBACxB,KAAK,EAAE,GAAG,CAAC,KAAK,IAAI,oBAAoB;oBACxC,GAAG,EAAE,GAAG,CAAC,GAAG;oBACZ,OAAO,EAAE,GAAG,CAAC,aAAa;iBAC3B,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,CAAC,EAAE,CAAC;gBACX,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,yBAAyB,EACjD,gEAAiE,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;gBAC1F,OAAO,IAAI,CAAC;YACd,CAAC;QACH,CAAC;QACD,KAAK,MAAM;YACT,OAAO,IAAI,qBAAqB,CAAC,GAAG,CAAC,GAAG,IAAI,gBAAgB,CAAC,CAAC;QAChE,KAAK,OAAO;YACV,8EAA8E;YAC9E,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,+BAA+B,EACvD,4DAA4D;gBAC5D,4EAA4E,CAAC,CAAC;YAChF,OAAO,IAAI,CAAC;QACd,KAAK,MAAM,CAAC;QACZ;YACE,OAAO,IAAI,CAAC;IAChB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { EmbeddingProvider, EmbedKind } from '../types.js';
|
|
2
|
+
export declare class FakeEmbeddingProvider implements EmbeddingProvider {
|
|
3
|
+
readonly id = "fake";
|
|
4
|
+
readonly model = "fake-hash";
|
|
5
|
+
readonly dim: number;
|
|
6
|
+
constructor(dim?: number);
|
|
7
|
+
embed(texts: string[], _kind: EmbedKind): Promise<Float32Array[]>;
|
|
8
|
+
}
|
|
9
|
+
//# sourceMappingURL=fake.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fake.d.ts","sourceRoot":"","sources":["../../../../src/engine/semantic/providers/fake.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAiChE,qBAAa,qBAAsB,YAAW,iBAAiB;IAC7D,QAAQ,CAAC,EAAE,UAAU;IACrB,QAAQ,CAAC,KAAK,eAAe;IAC7B,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;gBAET,GAAG,GAAE,MAAoB;IAI/B,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAGxE"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// Fake embedding provider (0.8.0) — a deterministic, offline, no-key provider
|
|
3
|
+
// for tests and local development. It is a real hashing vectorizer (the
|
|
4
|
+
// "hashing trick"): tokens map to fixed buckets, so identical text yields an
|
|
5
|
+
// identical vector (self-retrieval distance 0) and texts sharing words are
|
|
6
|
+
// closer. Not semantic in the learned sense, but deterministic and dependency-
|
|
7
|
+
// free — the injected provider tests use to exercise the full embed→vec→search
|
|
8
|
+
// loop without network or model weights.
|
|
9
|
+
// ============================================================================
|
|
10
|
+
const DEFAULT_DIM = 64;
|
|
11
|
+
// FNV-1a 32-bit — stable across runs/platforms, no crypto dep.
|
|
12
|
+
function fnv1a(s) {
|
|
13
|
+
let h = 0x811c9dc5;
|
|
14
|
+
for (let i = 0; i < s.length; i++) {
|
|
15
|
+
h ^= s.charCodeAt(i);
|
|
16
|
+
h = Math.imul(h, 0x01000193);
|
|
17
|
+
}
|
|
18
|
+
return h >>> 0;
|
|
19
|
+
}
|
|
20
|
+
function embedOne(text, dim) {
|
|
21
|
+
const v = new Float32Array(dim);
|
|
22
|
+
const tokens = text.toLowerCase().match(/[\p{L}\p{N}]+/gu) ?? [];
|
|
23
|
+
for (const tok of tokens) {
|
|
24
|
+
const h = fnv1a(tok);
|
|
25
|
+
const bucket = h % dim;
|
|
26
|
+
// signed update keeps the space from collapsing to all-positive
|
|
27
|
+
v[bucket] += (h & 1) === 0 ? 1 : -1;
|
|
28
|
+
}
|
|
29
|
+
// L2-normalize so cosine ranking via vec0's L2 distance is well-behaved;
|
|
30
|
+
// an empty/whitespace block maps to a stable nonzero unit vector.
|
|
31
|
+
let norm = 0;
|
|
32
|
+
for (let i = 0; i < dim; i++)
|
|
33
|
+
norm += v[i] * v[i];
|
|
34
|
+
norm = Math.sqrt(norm);
|
|
35
|
+
if (norm === 0) {
|
|
36
|
+
v[0] = 1;
|
|
37
|
+
return v;
|
|
38
|
+
}
|
|
39
|
+
for (let i = 0; i < dim; i++)
|
|
40
|
+
v[i] /= norm;
|
|
41
|
+
return v;
|
|
42
|
+
}
|
|
43
|
+
export class FakeEmbeddingProvider {
|
|
44
|
+
id = 'fake';
|
|
45
|
+
model = 'fake-hash';
|
|
46
|
+
dim;
|
|
47
|
+
constructor(dim = DEFAULT_DIM) {
|
|
48
|
+
this.dim = dim;
|
|
49
|
+
}
|
|
50
|
+
async embed(texts, _kind) {
|
|
51
|
+
return texts.map(t => embedOne(t, this.dim));
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=fake.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fake.js","sourceRoot":"","sources":["../../../../src/engine/semantic/providers/fake.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,8EAA8E;AAC9E,wEAAwE;AACxE,6EAA6E;AAC7E,2EAA2E;AAC3E,+EAA+E;AAC/E,+EAA+E;AAC/E,yCAAyC;AACzC,+EAA+E;AAI/E,MAAM,WAAW,GAAG,EAAE,CAAC;AAEvB,+DAA+D;AAC/D,SAAS,KAAK,CAAC,CAAS;IACtB,IAAI,CAAC,GAAG,UAAU,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,CAAC;IAC/B,CAAC;IACD,OAAO,CAAC,KAAK,CAAC,CAAC;AACjB,CAAC;AAED,SAAS,QAAQ,CAAC,IAAY,EAAE,GAAW;IACzC,MAAM,CAAC,GAAG,IAAI,YAAY,CAAC,GAAG,CAAC,CAAC;IAChC,MAAM,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,iBAAiB,CAAC,IAAI,EAAE,CAAC;IACjE,KAAK,MAAM,GAAG,IAAI,MAAM,EAAE,CAAC;QACzB,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACrB,MAAM,MAAM,GAAG,CAAC,GAAG,GAAG,CAAC;QACvB,gEAAgE;QAChE,CAAC,CAAC,MAAM,CAAE,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACvC,CAAC;IACD,yEAAyE;IACzE,kEAAkE;IAClE,IAAI,IAAI,GAAG,CAAC,CAAC;IACb,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;QAAE,IAAI,IAAI,CAAC,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC,CAAC,CAAE,CAAC;IACpD,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACvB,IAAI,IAAI,KAAK,CAAC,EAAE,CAAC;QAAC,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC;QAAC,OAAO,CAAC,CAAC;IAAC,CAAC;IACvC,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,GAAG,EAAE,CAAC,EAAE;QAAE,CAAC,CAAC,CAAC,CAAE,IAAI,IAAI,CAAC;IAC5C,OAAO,CAAC,CAAC;AACX,CAAC;AAED,MAAM,OAAO,qBAAqB;IACvB,EAAE,GAAG,MAAM,CAAC;IACZ,KAAK,GAAG,WAAW,CAAC;IACpB,GAAG,CAAS;IAErB,YAAY,MAAc,WAAW;QACnC,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe,EAAE,KAAgB;QAC3C,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC/C,CAAC;CACF"}
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import type { EmbeddingProvider, EmbedKind } from '../types.js';
|
|
2
|
+
export interface OpenAiProviderOptions {
|
|
3
|
+
apiKey: string;
|
|
4
|
+
model: string;
|
|
5
|
+
/** Explicit output dim. For 3-* models this is sent as `dimensions` (truncation). */
|
|
6
|
+
dim?: number | undefined;
|
|
7
|
+
/** API root (default https://api.openai.com); '/v1/embeddings' is appended. */
|
|
8
|
+
baseUrl?: string | undefined;
|
|
9
|
+
/** Injectable for tests; defaults to global fetch. */
|
|
10
|
+
fetchImpl?: typeof fetch | undefined;
|
|
11
|
+
}
|
|
12
|
+
export declare class OpenAiEmbeddingProvider implements EmbeddingProvider {
|
|
13
|
+
readonly id = "openai";
|
|
14
|
+
readonly model: string;
|
|
15
|
+
readonly dim: number;
|
|
16
|
+
private readonly apiKey;
|
|
17
|
+
private readonly url;
|
|
18
|
+
private readonly sendDimensions;
|
|
19
|
+
private readonly fetchImpl;
|
|
20
|
+
constructor(opts: OpenAiProviderOptions);
|
|
21
|
+
embed(texts: string[], _kind: EmbedKind): Promise<Float32Array[]>;
|
|
22
|
+
}
|
|
23
|
+
//# sourceMappingURL=openai.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai.d.ts","sourceRoot":"","sources":["../../../../src/engine/semantic/providers/openai.ts"],"names":[],"mappings":"AAOA,OAAO,KAAK,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM,aAAa,CAAC;AAYhE,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,qFAAqF;IACrF,GAAG,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IACzB,+EAA+E;IAC/E,OAAO,CAAC,EAAE,MAAM,GAAG,SAAS,CAAC;IAC7B,sDAAsD;IACtD,SAAS,CAAC,EAAE,OAAO,KAAK,GAAG,SAAS,CAAC;CACtC;AAMD,qBAAa,uBAAwB,YAAW,iBAAiB;IAC/D,QAAQ,CAAC,EAAE,YAAY;IACvB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,cAAc,CAAU;IACzC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAe;gBAE7B,IAAI,EAAE,qBAAqB;IAgCjC,KAAK,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,KAAK,EAAE,SAAS,GAAG,OAAO,CAAC,YAAY,EAAE,CAAC;CAkDxE"}
|
|
@@ -0,0 +1,103 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// OpenAI embedding provider (0.8.0) — a thin POST /v1/embeddings client over the
|
|
3
|
+
// built-in fetch (no SDK dep). Marginal-cost provider: if a deployment already
|
|
4
|
+
// has an OpenAI key wired, embeddings reuse the same key/base. The engine never
|
|
5
|
+
// stores the key beyond this provider closure.
|
|
6
|
+
// ============================================================================
|
|
7
|
+
const DEFAULT_BASE_URL = 'https://api.openai.com';
|
|
8
|
+
// Native output dims for the common models — used when no explicit dim is set.
|
|
9
|
+
// text-embedding-3-* additionally support server-side truncation via `dimensions`.
|
|
10
|
+
const MODEL_DIMS = {
|
|
11
|
+
'text-embedding-3-small': 1536,
|
|
12
|
+
'text-embedding-3-large': 3072,
|
|
13
|
+
'text-embedding-ada-002': 1536,
|
|
14
|
+
};
|
|
15
|
+
export class OpenAiEmbeddingProvider {
|
|
16
|
+
id = 'openai';
|
|
17
|
+
model;
|
|
18
|
+
dim;
|
|
19
|
+
apiKey;
|
|
20
|
+
url;
|
|
21
|
+
sendDimensions;
|
|
22
|
+
fetchImpl;
|
|
23
|
+
constructor(opts) {
|
|
24
|
+
if (!opts.apiKey)
|
|
25
|
+
throw new Error('OpenAI embedding provider requires an API key (MAAD_OPENAI_API_KEY)');
|
|
26
|
+
this.apiKey = opts.apiKey;
|
|
27
|
+
this.model = opts.model;
|
|
28
|
+
const native = MODEL_DIMS[opts.model];
|
|
29
|
+
const resolved = opts.dim ?? native;
|
|
30
|
+
if (resolved === undefined) {
|
|
31
|
+
throw new Error(`Cannot determine embedding dim for OpenAI model "${opts.model}". ` +
|
|
32
|
+
`Set MAAD_EMBED_DIM explicitly.`);
|
|
33
|
+
}
|
|
34
|
+
// Only text-embedding-3-* support server-side dimension truncation.
|
|
35
|
+
const supportsTruncation = /^text-embedding-3-/.test(opts.model);
|
|
36
|
+
if (opts.dim !== undefined && native !== undefined) {
|
|
37
|
+
if (opts.dim > native) {
|
|
38
|
+
throw new Error(`MAAD_EMBED_DIM=${opts.dim} exceeds the native dim ${native} for "${opts.model}"; ` +
|
|
39
|
+
`OpenAI embeddings can be truncated below native but never expanded.`);
|
|
40
|
+
}
|
|
41
|
+
if (opts.dim < native && !supportsTruncation) {
|
|
42
|
+
throw new Error(`Model "${opts.model}" does not support dimension truncation; ` +
|
|
43
|
+
`set MAAD_EMBED_DIM=${native} or omit it (only text-embedding-3-* can truncate).`);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
46
|
+
this.dim = resolved;
|
|
47
|
+
this.sendDimensions = supportsTruncation && opts.dim !== undefined && native !== undefined && opts.dim < native;
|
|
48
|
+
const base = (opts.baseUrl ?? DEFAULT_BASE_URL).replace(/\/+$/, '');
|
|
49
|
+
this.url = `${base}/v1/embeddings`;
|
|
50
|
+
this.fetchImpl = opts.fetchImpl ?? fetch;
|
|
51
|
+
}
|
|
52
|
+
async embed(texts, _kind) {
|
|
53
|
+
if (texts.length === 0)
|
|
54
|
+
return [];
|
|
55
|
+
const body = {
|
|
56
|
+
model: this.model,
|
|
57
|
+
input: texts,
|
|
58
|
+
encoding_format: 'float',
|
|
59
|
+
};
|
|
60
|
+
if (this.sendDimensions)
|
|
61
|
+
body['dimensions'] = this.dim;
|
|
62
|
+
const res = await this.fetchImpl(this.url, {
|
|
63
|
+
method: 'POST',
|
|
64
|
+
headers: {
|
|
65
|
+
'authorization': `Bearer ${this.apiKey}`,
|
|
66
|
+
'content-type': 'application/json',
|
|
67
|
+
},
|
|
68
|
+
body: JSON.stringify(body),
|
|
69
|
+
});
|
|
70
|
+
if (!res.ok) {
|
|
71
|
+
const detail = await res.text().catch(() => '');
|
|
72
|
+
throw new Error(`OpenAI embeddings request failed (${res.status}): ${detail.slice(0, 300)}`);
|
|
73
|
+
}
|
|
74
|
+
const json = (await res.json());
|
|
75
|
+
if (!json.data || json.data.length !== texts.length) {
|
|
76
|
+
throw new Error(`OpenAI embeddings returned ${json.data?.length ?? 0} vectors for ${texts.length} inputs`);
|
|
77
|
+
}
|
|
78
|
+
// Preserve input order regardless of response ordering. Validate each index
|
|
79
|
+
// (integer, in range, unique) and the vector width before use: an
|
|
80
|
+
// OpenAI-compatible endpoint (custom base URL) returning out-of-range or
|
|
81
|
+
// duplicate indices would otherwise leave holes in `out` that the worker's
|
|
82
|
+
// vectors[i]! trips over — failing and retrying the batch forever. A width
|
|
83
|
+
// mismatch would otherwise only surface as a sqlite-vec insert failure.
|
|
84
|
+
const out = new Array(texts.length);
|
|
85
|
+
const seen = new Set();
|
|
86
|
+
for (const item of json.data) {
|
|
87
|
+
if (!Number.isInteger(item.index) || item.index < 0 || item.index >= texts.length) {
|
|
88
|
+
throw new Error(`OpenAI embeddings returned an out-of-range index ${item.index} for ${texts.length} inputs.`);
|
|
89
|
+
}
|
|
90
|
+
if (seen.has(item.index)) {
|
|
91
|
+
throw new Error(`OpenAI embeddings returned a duplicate index ${item.index}.`);
|
|
92
|
+
}
|
|
93
|
+
seen.add(item.index);
|
|
94
|
+
if (item.embedding.length !== this.dim) {
|
|
95
|
+
throw new Error(`OpenAI returned a ${item.embedding.length}-dim vector but the provider is configured for ${this.dim} ` +
|
|
96
|
+
`(model "${this.model}"). Check MAAD_EMBED_DIM / MAAD_EMBED_MODEL.`);
|
|
97
|
+
}
|
|
98
|
+
out[item.index] = Float32Array.from(item.embedding);
|
|
99
|
+
}
|
|
100
|
+
return out;
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
//# sourceMappingURL=openai.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"openai.js","sourceRoot":"","sources":["../../../../src/engine/semantic/providers/openai.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,iFAAiF;AACjF,+EAA+E;AAC/E,gFAAgF;AAChF,+CAA+C;AAC/C,+EAA+E;AAI/E,MAAM,gBAAgB,GAAG,wBAAwB,CAAC;AAElD,+EAA+E;AAC/E,mFAAmF;AACnF,MAAM,UAAU,GAA2B;IACzC,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;IAC9B,wBAAwB,EAAE,IAAI;CAC/B,CAAC;AAiBF,MAAM,OAAO,uBAAuB;IACzB,EAAE,GAAG,QAAQ,CAAC;IACd,KAAK,CAAS;IACd,GAAG,CAAS;IACJ,MAAM,CAAS;IACf,GAAG,CAAS;IACZ,cAAc,CAAU;IACxB,SAAS,CAAe;IAEzC,YAAY,IAA2B;QACrC,IAAI,CAAC,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;QACzG,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;QACxB,MAAM,MAAM,GAAG,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACtC,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,IAAI,MAAM,CAAC;QACpC,IAAI,QAAQ,KAAK,SAAS,EAAE,CAAC;YAC3B,MAAM,IAAI,KAAK,CACb,oDAAoD,IAAI,CAAC,KAAK,KAAK;gBACnE,gCAAgC,CAAC,CAAC;QACtC,CAAC;QACD,oEAAoE;QACpE,MAAM,kBAAkB,GAAG,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjE,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;YACnD,IAAI,IAAI,CAAC,GAAG,GAAG,MAAM,EAAE,CAAC;gBACtB,MAAM,IAAI,KAAK,CACb,kBAAkB,IAAI,CAAC,GAAG,2BAA2B,MAAM,SAAS,IAAI,CAAC,KAAK,KAAK;oBACnF,qEAAqE,CAAC,CAAC;YAC3E,CAAC;YACD,IAAI,IAAI,CAAC,GAAG,GAAG,MAAM,IAAI,CAAC,kBAAkB,EAAE,CAAC;gBAC7C,MAAM,IAAI,KAAK,CACb,UAAU,IAAI,CAAC,KAAK,2CAA2C;oBAC/D,sBAAsB,MAAM,qDAAqD,CAAC,CAAC;YACvF,CAAC;QACH,CAAC;QACD,IAAI,CAAC,GAAG,GAAG,QAAQ,CAAC;QACpB,IAAI,CAAC,cAAc,GAAG,kBAAkB,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,IAAI,MAAM,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,GAAG,MAAM,CAAC;QAChH,MAAM,IAAI,GAAG,CAAC,IAAI,CAAC,OAAO,IAAI,gBAAgB,CAAC,CAAC,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;QACpE,IAAI,CAAC,GAAG,GAAG,GAAG,IAAI,gBAAgB,CAAC;QACnC,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;IAC3C,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,KAAe,EAAE,KAAgB;QAC3C,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC;YAAE,OAAO,EAAE,CAAC;QAClC,MAAM,IAAI,GAA4B;YACpC,KAAK,EAAE,IAAI,CAAC,KAAK;YACjB,KAAK,EAAE,KAAK;YACZ,eAAe,EAAE,OAAO;SACzB,CAAC;QACF,IAAI,IAAI,CAAC,cAAc;YAAE,IAAI,CAAC,YAAY,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC;QAEvD,MAAM,GAAG,GAAG,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,GAAG,EAAE;YACzC,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,eAAe,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;gBACxC,cAAc,EAAE,kBAAkB;aACnC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC;SAC3B,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,MAAM,GAAG,MAAM,GAAG,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,CAAC,EAAE,CAAC,CAAC;YAChD,MAAM,IAAI,KAAK,CAAC,qCAAqC,GAAG,CAAC,MAAM,MAAM,MAAM,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,EAAE,CAAC,CAAC;QAC/F,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAAuB,CAAC;QACtD,IAAI,CAAC,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,MAAM,KAAK,KAAK,CAAC,MAAM,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CAAC,8BAA8B,IAAI,CAAC,IAAI,EAAE,MAAM,IAAI,CAAC,gBAAgB,KAAK,CAAC,MAAM,SAAS,CAAC,CAAC;QAC7G,CAAC;QACD,4EAA4E;QAC5E,kEAAkE;QAClE,yEAAyE;QACzE,2EAA2E;QAC3E,2EAA2E;QAC3E,wEAAwE;QACxE,MAAM,GAAG,GAAmB,IAAI,KAAK,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;QACpD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,KAAK,MAAM,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;YAC7B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,CAAC,KAAK,GAAG,CAAC,IAAI,IAAI,CAAC,KAAK,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;gBAClF,MAAM,IAAI,KAAK,CAAC,oDAAoD,IAAI,CAAC,KAAK,QAAQ,KAAK,CAAC,MAAM,UAAU,CAAC,CAAC;YAChH,CAAC;YACD,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzB,MAAM,IAAI,KAAK,CAAC,gDAAgD,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;YACjF,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;YACrB,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;gBACvC,MAAM,IAAI,KAAK,CACb,qBAAqB,IAAI,CAAC,SAAS,CAAC,MAAM,kDAAkD,IAAI,CAAC,GAAG,GAAG;oBACvG,WAAW,IAAI,CAAC,KAAK,8CAA8C,CAAC,CAAC;YACzE,CAAC;YACD,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QACtD,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF"}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
/** Default RRF constant. 60 is the value from the original RRF paper; larger K
|
|
2
|
+
* flattens the contribution of top ranks (less aggressive). */
|
|
3
|
+
export declare const DEFAULT_RRF_K = 60;
|
|
4
|
+
/**
|
|
5
|
+
* Fuse ranked lists of opaque string keys. Input lists are in rank order (index
|
|
6
|
+
* 0 = best). Returns a key→score map; higher score = more relevant.
|
|
7
|
+
*/
|
|
8
|
+
export declare function rrfFuse(lists: string[][], k?: number): Map<string, number>;
|
|
9
|
+
//# sourceMappingURL=rrf.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rrf.d.ts","sourceRoot":"","sources":["../../../src/engine/semantic/rrf.ts"],"names":[],"mappings":"AAWA;gEACgE;AAChE,eAAO,MAAM,aAAa,KAAK,CAAC;AAEhC;;;GAGG;AACH,wBAAgB,OAAO,CAAC,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE,CAAC,GAAE,MAAsB,GAAG,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAUzF"}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
// ============================================================================
|
|
2
|
+
// Reciprocal Rank Fusion (0.8.0)
|
|
3
|
+
// Merge N independently-ranked lists into one score map. RRF is rank-based and
|
|
4
|
+
// scale-free — exactly why the engine fuses here rather than handing the agent
|
|
5
|
+
// two incomparable score columns (BM25 vs cosine distance). Each item's score
|
|
6
|
+
// is Σ over the lists it appears in of 1/(K + rank), rank 1-based.
|
|
7
|
+
//
|
|
8
|
+
// Used for hybrid (two legs). Called with a single list it degrades to a clean
|
|
9
|
+
// rank-based score, so exact/semantic single-leg paths reuse the same merge.
|
|
10
|
+
// ============================================================================
|
|
11
|
+
/** Default RRF constant. 60 is the value from the original RRF paper; larger K
|
|
12
|
+
* flattens the contribution of top ranks (less aggressive). */
|
|
13
|
+
export const DEFAULT_RRF_K = 60;
|
|
14
|
+
/**
|
|
15
|
+
* Fuse ranked lists of opaque string keys. Input lists are in rank order (index
|
|
16
|
+
* 0 = best). Returns a key→score map; higher score = more relevant.
|
|
17
|
+
*/
|
|
18
|
+
export function rrfFuse(lists, k = DEFAULT_RRF_K) {
|
|
19
|
+
const scores = new Map();
|
|
20
|
+
for (const list of lists) {
|
|
21
|
+
for (let i = 0; i < list.length; i++) {
|
|
22
|
+
const key = list[i];
|
|
23
|
+
const contribution = 1 / (k + i + 1);
|
|
24
|
+
scores.set(key, (scores.get(key) ?? 0) + contribution);
|
|
25
|
+
}
|
|
26
|
+
}
|
|
27
|
+
return scores;
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=rrf.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rrf.js","sourceRoot":"","sources":["../../../src/engine/semantic/rrf.ts"],"names":[],"mappings":"AAAA,+EAA+E;AAC/E,iCAAiC;AACjC,+EAA+E;AAC/E,+EAA+E;AAC/E,8EAA8E;AAC9E,mEAAmE;AACnE,EAAE;AACF,+EAA+E;AAC/E,6EAA6E;AAC7E,+EAA+E;AAE/E;gEACgE;AAChE,MAAM,CAAC,MAAM,aAAa,GAAG,EAAE,CAAC;AAEhC;;;GAGG;AACH,MAAM,UAAU,OAAO,CAAC,KAAiB,EAAE,IAAY,aAAa;IAClE,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;QACzB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACrC,MAAM,GAAG,GAAG,IAAI,CAAC,CAAC,CAAE,CAAC;YACrB,MAAM,YAAY,GAAG,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC;YACrC,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,YAAY,CAAC,CAAC;QACzD,CAAC;IACH,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
import { type Result } from '../../errors.js';
|
|
2
|
+
import type { EngineContext } from '../context.js';
|
|
3
|
+
import type { SemanticSearchQuery, SemanticSearchResult, SearchMode } from './types.js';
|
|
4
|
+
export declare function isValidMode(mode: string): mode is SearchMode;
|
|
5
|
+
export declare function semanticSearch(ctx: EngineContext, query: SemanticSearchQuery): Promise<Result<SemanticSearchResult>>;
|
|
6
|
+
//# sourceMappingURL=search.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"search.d.ts","sourceRoot":"","sources":["../../../src/engine/semantic/search.ts"],"names":[],"mappings":"AAUA,OAAO,EAAiB,KAAK,MAAM,EAAE,MAAM,iBAAiB,CAAC;AAE7D,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAInD,OAAO,KAAK,EACV,mBAAmB,EACnB,oBAAoB,EAEpB,UAAU,EACX,MAAM,YAAY,CAAC;AAapB,wBAAgB,WAAW,CAAC,IAAI,EAAE,MAAM,GAAG,IAAI,IAAI,UAAU,CAE5D;AAcD,wBAAsB,cAAc,CAClC,GAAG,EAAE,aAAa,EAClB,KAAK,EAAE,mBAAmB,GACzB,OAAO,CAAC,MAAM,CAAC,oBAAoB,CAAC,CAAC,CAmKvC"}
|