@getrift/rift 0.1.0-beta.1 → 0.1.0-beta.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +154 -0
- package/dist/src/cli/status/friend-header.d.ts.map +1 -1
- package/dist/src/cli/status/friend-header.js +31 -5
- package/dist/src/cli/status/friend-header.js.map +1 -1
- package/dist/src/ingestion/indexer.d.ts +7 -0
- package/dist/src/ingestion/indexer.d.ts.map +1 -1
- package/dist/src/ingestion/indexer.js +36 -2
- package/dist/src/ingestion/indexer.js.map +1 -1
- package/dist/src/ingestion/skip-quarantine.d.ts +10 -0
- package/dist/src/ingestion/skip-quarantine.d.ts.map +1 -0
- package/dist/src/ingestion/skip-quarantine.js +35 -0
- package/dist/src/ingestion/skip-quarantine.js.map +1 -0
- package/dist/src/jobs/handlers/compact.d.ts.map +1 -1
- package/dist/src/jobs/handlers/compact.js +25 -4
- package/dist/src/jobs/handlers/compact.js.map +1 -1
- package/dist/src/jobs/handlers/ingest.d.ts.map +1 -1
- package/dist/src/jobs/handlers/ingest.js +26 -6
- package/dist/src/jobs/handlers/ingest.js.map +1 -1
- package/dist/src/jobs/handlers/reconcile.d.ts.map +1 -1
- package/dist/src/jobs/handlers/reconcile.js +30 -8
- package/dist/src/jobs/handlers/reconcile.js.map +1 -1
- package/dist/src/jobs/handlers/reindex.d.ts.map +1 -1
- package/dist/src/jobs/handlers/reindex.js +1 -1
- package/dist/src/jobs/handlers/reindex.js.map +1 -1
- package/dist/src/jobs/handlers/save.d.ts.map +1 -1
- package/dist/src/jobs/handlers/save.js +9 -2
- package/dist/src/jobs/handlers/save.js.map +1 -1
- package/dist/src/jobs/worker-entry.d.ts.map +1 -1
- package/dist/src/jobs/worker-entry.js +2 -0
- package/dist/src/jobs/worker-entry.js.map +1 -1
- package/dist/src/main.js +2 -0
- package/dist/src/main.js.map +1 -1
- package/dist/src/observability/embedding-events.d.ts +52 -0
- package/dist/src/observability/embedding-events.d.ts.map +1 -0
- package/dist/src/observability/embedding-events.js +149 -0
- package/dist/src/observability/embedding-events.js.map +1 -0
- package/dist/src/observability/index-events.d.ts +70 -0
- package/dist/src/observability/index-events.d.ts.map +1 -0
- package/dist/src/observability/index-events.js +148 -0
- package/dist/src/observability/index-events.js.map +1 -0
- package/dist/src/providers/ollama-embed.d.ts +2 -1
- package/dist/src/providers/ollama-embed.d.ts.map +1 -1
- package/dist/src/providers/ollama-embed.js +1 -0
- package/dist/src/providers/ollama-embed.js.map +1 -1
- package/dist/src/providers/stub.d.ts +2 -0
- package/dist/src/providers/stub.d.ts.map +1 -1
- package/dist/src/providers/stub.js +2 -0
- package/dist/src/providers/stub.js.map +1 -1
- package/dist/src/providers/types.d.ts +11 -0
- package/dist/src/providers/types.d.ts.map +1 -1
- package/dist/src/providers/voyage.d.ts +2 -1
- package/dist/src/providers/voyage.d.ts.map +1 -1
- package/dist/src/providers/voyage.js +1 -0
- package/dist/src/providers/voyage.js.map +1 -1
- package/dist/src/server/routes/friend-status.d.ts +71 -2
- package/dist/src/server/routes/friend-status.d.ts.map +1 -1
- package/dist/src/server/routes/friend-status.js +116 -7
- package/dist/src/server/routes/friend-status.js.map +1 -1
- package/dist/src/storage/rebuild.d.ts +1 -1
- package/dist/src/storage/rebuild.d.ts.map +1 -1
- package/dist/src/storage/rebuild.js +90 -32
- package/dist/src/storage/rebuild.js.map +1 -1
- package/package.json +1 -1
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Embedding-events lane — append-only JSONL.
|
|
3
|
+
*
|
|
4
|
+
* One row per call to an embedding provider on an *ingest* code path
|
|
5
|
+
* (watcher / scheduled-scan / reconcile / ingest-job / compact / backfill).
|
|
6
|
+
* Search-time embedding calls (search routes, hybrid retrieval) are
|
|
7
|
+
* deliberately NOT logged here — otherwise a user search would refresh
|
|
8
|
+
* `voyage.last_embed_at` and mask an ingestion outage.
|
|
9
|
+
*
|
|
10
|
+
* Provider-agnostic on purpose: `provider` and `model` are payload
|
|
11
|
+
* fields, not the file name, so a future Ollama/local-embedding swap
|
|
12
|
+
* shares the same lane and the same `/status/friend` consumer.
|
|
13
|
+
*
|
|
14
|
+
* Failure mode: writes are best-effort and never throw — embedding must
|
|
15
|
+
* not be observably blocked by telemetry. Reads are tolerant of bad
|
|
16
|
+
* lines so one corrupt row doesn't blank the status dashboard.
|
|
17
|
+
*/
|
|
18
|
+
import fs from "node:fs";
|
|
19
|
+
import path from "node:path";
|
|
20
|
+
export const EMBEDDING_PIPELINES = [
|
|
21
|
+
"watcher",
|
|
22
|
+
"scheduled_scan",
|
|
23
|
+
"reconcile",
|
|
24
|
+
"ingest_job",
|
|
25
|
+
"compact",
|
|
26
|
+
"backfill",
|
|
27
|
+
"capture",
|
|
28
|
+
"reindex",
|
|
29
|
+
];
|
|
30
|
+
const MAX_ERROR_MESSAGE_LEN = 512;
|
|
31
|
+
export function eventsFilePath(dataDir) {
|
|
32
|
+
return path.join(dataDir, "observability", "embedding-events.jsonl");
|
|
33
|
+
}
|
|
34
|
+
/**
|
|
35
|
+
* Append a single event to the lane. Never throws — telemetry must not
|
|
36
|
+
* break embedding. On filesystem error, writes a single stderr line so
|
|
37
|
+
* the failure is at least visible to operators tailing logs.
|
|
38
|
+
*/
|
|
39
|
+
export function appendEmbeddingEvent(dataDir, event) {
|
|
40
|
+
const filePath = eventsFilePath(dataDir);
|
|
41
|
+
try {
|
|
42
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
43
|
+
fs.appendFileSync(filePath, JSON.stringify(event) + "\n");
|
|
44
|
+
}
|
|
45
|
+
catch (err) {
|
|
46
|
+
process.stderr.write(`[embedding-events] write failed: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Run an embedding call and record one event. Re-throws errors so the
|
|
51
|
+
* caller's existing failure-handling path is unchanged; logging is a
|
|
52
|
+
* pure side effect.
|
|
53
|
+
*
|
|
54
|
+
* Use this at every ingest call site. Do NOT use it for search-time
|
|
55
|
+
* embedding — keeping search out of the lane is what makes
|
|
56
|
+
* `last_embed_at` meaningful.
|
|
57
|
+
*/
|
|
58
|
+
export async function recordEmbed(dataDir, provider, ctx, fn) {
|
|
59
|
+
const start = Date.now();
|
|
60
|
+
try {
|
|
61
|
+
const result = await fn();
|
|
62
|
+
appendEmbeddingEvent(dataDir, {
|
|
63
|
+
ts: new Date().toISOString(),
|
|
64
|
+
provider: provider.name,
|
|
65
|
+
model: provider.model,
|
|
66
|
+
pipeline: ctx.pipeline,
|
|
67
|
+
operation: ctx.operation,
|
|
68
|
+
outcome: "success",
|
|
69
|
+
input_count: ctx.input_count,
|
|
70
|
+
ms: Date.now() - start,
|
|
71
|
+
});
|
|
72
|
+
return result;
|
|
73
|
+
}
|
|
74
|
+
catch (err) {
|
|
75
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
76
|
+
appendEmbeddingEvent(dataDir, {
|
|
77
|
+
ts: new Date().toISOString(),
|
|
78
|
+
provider: provider.name,
|
|
79
|
+
model: provider.model,
|
|
80
|
+
pipeline: ctx.pipeline,
|
|
81
|
+
operation: ctx.operation,
|
|
82
|
+
outcome: "error",
|
|
83
|
+
input_count: ctx.input_count,
|
|
84
|
+
ms: Date.now() - start,
|
|
85
|
+
error_class: classifyError(err),
|
|
86
|
+
error_message: message.slice(0, MAX_ERROR_MESSAGE_LEN),
|
|
87
|
+
});
|
|
88
|
+
throw err;
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
/**
|
|
92
|
+
* Best-effort classification for the `error_class` field.
|
|
93
|
+
* Voyage / Ollama errors carry `Voyage API <code>: ...` /
|
|
94
|
+
* `Ollama embed API <code>: ...` shapes — extract the status code so
|
|
95
|
+
* status views can group by class without parsing free-text.
|
|
96
|
+
*/
|
|
97
|
+
function classifyError(err) {
|
|
98
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
99
|
+
const httpMatch = msg.match(/\b(\d{3})\b/);
|
|
100
|
+
if (httpMatch && msg.toLowerCase().includes("api")) {
|
|
101
|
+
return `provider_${httpMatch[1]}`;
|
|
102
|
+
}
|
|
103
|
+
if (err instanceof TypeError)
|
|
104
|
+
return "network";
|
|
105
|
+
return "unknown";
|
|
106
|
+
}
|
|
107
|
+
/**
|
|
108
|
+
* Tolerant reader. One bad JSON line is skipped; the rest are returned.
|
|
109
|
+
* Returns [] if the file is missing.
|
|
110
|
+
*/
|
|
111
|
+
export function readEmbeddingEvents(dataDir) {
|
|
112
|
+
let raw;
|
|
113
|
+
try {
|
|
114
|
+
raw = fs.readFileSync(eventsFilePath(dataDir), "utf-8");
|
|
115
|
+
}
|
|
116
|
+
catch {
|
|
117
|
+
return [];
|
|
118
|
+
}
|
|
119
|
+
const out = [];
|
|
120
|
+
for (const line of raw.split("\n")) {
|
|
121
|
+
if (!line)
|
|
122
|
+
continue;
|
|
123
|
+
let parsed;
|
|
124
|
+
try {
|
|
125
|
+
parsed = JSON.parse(line);
|
|
126
|
+
}
|
|
127
|
+
catch {
|
|
128
|
+
continue;
|
|
129
|
+
}
|
|
130
|
+
if (!isEmbeddingEvent(parsed))
|
|
131
|
+
continue;
|
|
132
|
+
out.push(parsed);
|
|
133
|
+
}
|
|
134
|
+
return out;
|
|
135
|
+
}
|
|
136
|
+
function isEmbeddingEvent(value) {
|
|
137
|
+
if (value === null || typeof value !== "object")
|
|
138
|
+
return false;
|
|
139
|
+
const v = value;
|
|
140
|
+
return (typeof v["ts"] === "string" &&
|
|
141
|
+
typeof v["provider"] === "string" &&
|
|
142
|
+
typeof v["model"] === "string" &&
|
|
143
|
+
typeof v["pipeline"] === "string" &&
|
|
144
|
+
typeof v["operation"] === "string" &&
|
|
145
|
+
(v["outcome"] === "success" || v["outcome"] === "error") &&
|
|
146
|
+
typeof v["input_count"] === "number" &&
|
|
147
|
+
typeof v["ms"] === "number");
|
|
148
|
+
}
|
|
149
|
+
//# sourceMappingURL=embedding-events.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"embedding-events.js","sourceRoot":"","sources":["../../../src/observability/embedding-events.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAG7B,MAAM,CAAC,MAAM,mBAAmB,GAAG;IACjC,SAAS;IACT,gBAAgB;IAChB,WAAW;IACX,YAAY;IACZ,SAAS;IACT,UAAU;IACV,SAAS;IACT,SAAS;CACD,CAAC;AAyBX,MAAM,qBAAqB,GAAG,GAAG,CAAC;AASlC,MAAM,UAAU,cAAc,CAAC,OAAe;IAC5C,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,EAAE,wBAAwB,CAAC,CAAC;AACvE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,oBAAoB,CAClC,OAAe,EACf,KAAqB;IAErB,MAAM,QAAQ,GAAG,cAAc,CAAC,OAAO,CAAC,CAAC;IACzC,IAAI,CAAC;QACH,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,EAAE,CAAC,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;IAC5D,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,oCAAoC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CACzF,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,WAAW,CAC/B,OAAe,EACf,QAA2B,EAC3B,GAAiB,EACjB,EAAoB;IAEpB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,EAAE,EAAE,CAAC;QAC1B,oBAAoB,CAAC,OAAO,EAAE;YAC5B,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC5B,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,OAAO,EAAE,SAAS;YAClB,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;SACvB,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,oBAAoB,CAAC,OAAO,EAAE;YAC5B,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC5B,QAAQ,EAAE,QAAQ,CAAC,IAAI;YACvB,KAAK,EAAE,QAAQ,CAAC,KAAK;YACrB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,OAAO,EAAE,OAAO;YAChB,WAAW,EAAE,GAAG,CAAC,WAAW;YAC5B,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YACtB,WAAW,EAAE,aAAa,CAAC,GAAG,CAAC;YAC/B,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC;SACvD,CAAC,CAAC;QACH,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;;;GAKG;AACH,SAAS,aAAa,CAAC,GAAY;IACjC,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC7D,MAAM,SAAS,GAAG,GAAG,CAAC,KAAK,CAAC,aAAa,CAAC,CAAC;IAC3C,IAAI,SAAS,IAAI,GAAG,CAAC,WAAW,EAAE,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QACnD,OAAO,YAAY,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC;IACpC,CAAC;IACD,IAAI,GAAG,YAAY,SAAS;QAAE,OAAO,SAAS,CAAC;IAC/C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAe;IACjD,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,cAAc,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,GAAG,GAAqB,EAAE,CAAC;IACjC,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,CAAC,gBAAgB,CAAC,MAAM,CAAC;YAAE,SAAS;QACxC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,gBAAgB,CAAC,KAAc;IACtC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC9D,MAAM,CAAC,GAAG,KAAgC,CAAC;IAC3C,OAAO,CACL,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,QAAQ;QAC3B,OAAO,CAAC,CAAC,UAAU,CAAC,KAAK,QAAQ;QACjC,OAAO,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ;QAC9B,OAAO,CAAC,CAAC,UAAU,CAAC,KAAK,QAAQ;QACjC,OAAO,CAAC,CAAC,WAAW,CAAC,KAAK,QAAQ;QAClC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,SAAS,CAAC,KAAK,OAAO,CAAC;QACxD,OAAO,CAAC,CAAC,aAAa,CAAC,KAAK,QAAQ;QACpC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,QAAQ,CAC5B,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import type { EmbeddingPipeline } from "./embedding-events.js";
|
|
2
|
+
export type IndexOutcome = "success" | "error";
|
|
3
|
+
export interface IndexEvent {
|
|
4
|
+
ts: string;
|
|
5
|
+
/**
|
|
6
|
+
* Target LanceDB table. Free-form string (not `TableName`) so shadow
|
|
7
|
+
* writes can record their target without coupling to the live table
|
|
8
|
+
* name set.
|
|
9
|
+
*/
|
|
10
|
+
table: string;
|
|
11
|
+
pipeline: EmbeddingPipeline;
|
|
12
|
+
/**
|
|
13
|
+
* Free-form caller label (e.g. "structured_doc_upsert",
|
|
14
|
+
* "conversation_upsert", "digest_upsert"). Helps debug "which write
|
|
15
|
+
* path"; does not gate any status logic today.
|
|
16
|
+
*/
|
|
17
|
+
operation: string;
|
|
18
|
+
outcome: IndexOutcome;
|
|
19
|
+
/** Number of rows in this `table.add(...)` call (or rollback batch). */
|
|
20
|
+
row_count: number;
|
|
21
|
+
ms: number;
|
|
22
|
+
/**
|
|
23
|
+
* Lifecycle phase.
|
|
24
|
+
* - "live" (default when omitted) — the row is now searchable.
|
|
25
|
+
* Direct writes are always "live"; shadow rebuilds emit a single
|
|
26
|
+
* "live" event per table after `commitAllSwaps` succeeds.
|
|
27
|
+
* - "shadow" — the row was written to a pre-commit shadow table. If
|
|
28
|
+
* the subsequent swap fails the row is NOT searchable, so
|
|
29
|
+
* `computeIndexHealth` does NOT credit `last_update_at` from
|
|
30
|
+
* shadow successes. Shadow errors still surface as `last_error_at`
|
|
31
|
+
* because they represent real LanceDB write failures regardless of
|
|
32
|
+
* the eventual swap outcome.
|
|
33
|
+
*/
|
|
34
|
+
phase?: "live" | "shadow";
|
|
35
|
+
/** Compact error class. Present when outcome === "error". */
|
|
36
|
+
error_class?: string;
|
|
37
|
+
/** Truncated error message (≤ 512 chars). Present when outcome === "error". */
|
|
38
|
+
error_message?: string;
|
|
39
|
+
}
|
|
40
|
+
export interface IndexWriteContext {
|
|
41
|
+
table: string;
|
|
42
|
+
pipeline: EmbeddingPipeline;
|
|
43
|
+
operation: string;
|
|
44
|
+
row_count: number;
|
|
45
|
+
/** Defaults to "live". Set "shadow" for pre-commit shadow-table writes. */
|
|
46
|
+
phase?: "live" | "shadow";
|
|
47
|
+
}
|
|
48
|
+
export declare function indexEventsFilePath(dataDir: string): string;
|
|
49
|
+
/**
|
|
50
|
+
* Append a single event to the lane. Never throws — telemetry must not
|
|
51
|
+
* break indexing. On filesystem error, writes a single stderr line so
|
|
52
|
+
* the failure is at least visible to operators tailing logs.
|
|
53
|
+
*/
|
|
54
|
+
export declare function appendIndexEvent(dataDir: string, event: IndexEvent): void;
|
|
55
|
+
/**
|
|
56
|
+
* Run a LanceDB write and record one event. Re-throws errors so the
|
|
57
|
+
* caller's existing failure-handling path is unchanged; logging is a
|
|
58
|
+
* pure side effect.
|
|
59
|
+
*
|
|
60
|
+
* Use this only at write boundaries (`table.add(...)`). Do NOT use it
|
|
61
|
+
* for reads, queries, or deletes — keeping the lane write-only is what
|
|
62
|
+
* makes `last_update_at` meaningful.
|
|
63
|
+
*/
|
|
64
|
+
export declare function recordIndexWrite<T>(dataDir: string, ctx: IndexWriteContext, fn: () => Promise<T>): Promise<T>;
|
|
65
|
+
/**
|
|
66
|
+
* Tolerant reader. One bad JSON line is skipped; the rest are returned.
|
|
67
|
+
* Returns [] if the file is missing.
|
|
68
|
+
*/
|
|
69
|
+
export declare function readIndexEvents(dataDir: string): IndexEvent[];
|
|
70
|
+
//# sourceMappingURL=index-events.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-events.d.ts","sourceRoot":"","sources":["../../../src/observability/index-events.ts"],"names":[],"mappings":"AAmBA,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,uBAAuB,CAAC;AAE/D,MAAM,MAAM,YAAY,GAAG,SAAS,GAAG,OAAO,CAAC;AAE/C,MAAM,WAAW,UAAU;IACzB,EAAE,EAAE,MAAM,CAAC;IACX;;;;OAIG;IACH,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,iBAAiB,CAAC;IAC5B;;;;OAIG;IACH,SAAS,EAAE,MAAM,CAAC;IAClB,OAAO,EAAE,YAAY,CAAC;IACtB,wEAAwE;IACxE,SAAS,EAAE,MAAM,CAAC;IAClB,EAAE,EAAE,MAAM,CAAC;IACX;;;;;;;;;;;OAWG;IACH,KAAK,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC;IAC1B,6DAA6D;IAC7D,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,+EAA+E;IAC/E,aAAa,CAAC,EAAE,MAAM,CAAC;CACxB;AAID,MAAM,WAAW,iBAAiB;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,2EAA2E;IAC3E,KAAK,CAAC,EAAE,MAAM,GAAG,QAAQ,CAAC;CAC3B;AAED,wBAAgB,mBAAmB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAE3D;AAED;;;;GAIG;AACH,wBAAgB,gBAAgB,CAC9B,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,UAAU,GAChB,IAAI,CAUN;AAED;;;;;;;;GAQG;AACH,wBAAsB,gBAAgB,CAAC,CAAC,EACtC,OAAO,EAAE,MAAM,EACf,GAAG,EAAE,iBAAiB,EACtB,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GACnB,OAAO,CAAC,CAAC,CAAC,CAgCZ;AAmBD;;;GAGG;AACH,wBAAgB,eAAe,CAAC,OAAO,EAAE,MAAM,GAAG,UAAU,EAAE,CAoB7D"}
|
|
@@ -0,0 +1,148 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Index-events lane — append-only JSONL.
|
|
3
|
+
*
|
|
4
|
+
* One row per write boundary against LanceDB (`table.add(...)`). This
|
|
5
|
+
* complements `embedding-events.jsonl`: the embedding lane proves the
|
|
6
|
+
* vector was produced, this lane proves the row was actually persisted
|
|
7
|
+
* to LanceDB and is searchable. Without it, `/status/friend` cannot
|
|
8
|
+
* distinguish "embedded but not indexed yet" from "fully searchable".
|
|
9
|
+
*
|
|
10
|
+
* Provider-agnostic and table-agnostic: `table` and `pipeline` are
|
|
11
|
+
* payload fields. Reads / search queries are deliberately NOT wrapped —
|
|
12
|
+
* they're not write events and would muddy `index.last_update_at`.
|
|
13
|
+
*
|
|
14
|
+
* Failure mode: writes are best-effort and never throw — observability
|
|
15
|
+
* must not break indexing. The reader is tolerant of bad lines so one
|
|
16
|
+
* corrupt row doesn't blank the status dashboard.
|
|
17
|
+
*/
|
|
18
|
+
import fs from "node:fs";
|
|
19
|
+
import path from "node:path";
|
|
20
|
+
const MAX_ERROR_MESSAGE_LEN = 512;
|
|
21
|
+
export function indexEventsFilePath(dataDir) {
|
|
22
|
+
return path.join(dataDir, "observability", "index-events.jsonl");
|
|
23
|
+
}
|
|
24
|
+
/**
|
|
25
|
+
* Append a single event to the lane. Never throws — telemetry must not
|
|
26
|
+
* break indexing. On filesystem error, writes a single stderr line so
|
|
27
|
+
* the failure is at least visible to operators tailing logs.
|
|
28
|
+
*/
|
|
29
|
+
export function appendIndexEvent(dataDir, event) {
|
|
30
|
+
const filePath = indexEventsFilePath(dataDir);
|
|
31
|
+
try {
|
|
32
|
+
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
33
|
+
fs.appendFileSync(filePath, JSON.stringify(event) + "\n");
|
|
34
|
+
}
|
|
35
|
+
catch (err) {
|
|
36
|
+
process.stderr.write(`[index-events] write failed: ${err instanceof Error ? err.message : String(err)}\n`);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Run a LanceDB write and record one event. Re-throws errors so the
|
|
41
|
+
* caller's existing failure-handling path is unchanged; logging is a
|
|
42
|
+
* pure side effect.
|
|
43
|
+
*
|
|
44
|
+
* Use this only at write boundaries (`table.add(...)`). Do NOT use it
|
|
45
|
+
* for reads, queries, or deletes — keeping the lane write-only is what
|
|
46
|
+
* makes `last_update_at` meaningful.
|
|
47
|
+
*/
|
|
48
|
+
export async function recordIndexWrite(dataDir, ctx, fn) {
|
|
49
|
+
const start = Date.now();
|
|
50
|
+
const phase = ctx.phase ?? "live";
|
|
51
|
+
try {
|
|
52
|
+
const result = await fn();
|
|
53
|
+
appendIndexEvent(dataDir, {
|
|
54
|
+
ts: new Date().toISOString(),
|
|
55
|
+
table: ctx.table,
|
|
56
|
+
pipeline: ctx.pipeline,
|
|
57
|
+
operation: ctx.operation,
|
|
58
|
+
outcome: "success",
|
|
59
|
+
row_count: ctx.row_count,
|
|
60
|
+
ms: Date.now() - start,
|
|
61
|
+
phase,
|
|
62
|
+
});
|
|
63
|
+
return result;
|
|
64
|
+
}
|
|
65
|
+
catch (err) {
|
|
66
|
+
const message = err instanceof Error ? err.message : String(err);
|
|
67
|
+
appendIndexEvent(dataDir, {
|
|
68
|
+
ts: new Date().toISOString(),
|
|
69
|
+
table: ctx.table,
|
|
70
|
+
pipeline: ctx.pipeline,
|
|
71
|
+
operation: ctx.operation,
|
|
72
|
+
outcome: "error",
|
|
73
|
+
row_count: ctx.row_count,
|
|
74
|
+
ms: Date.now() - start,
|
|
75
|
+
phase,
|
|
76
|
+
error_class: classifyError(err),
|
|
77
|
+
error_message: message.slice(0, MAX_ERROR_MESSAGE_LEN),
|
|
78
|
+
});
|
|
79
|
+
throw err;
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
/**
|
|
83
|
+
* Best-effort classification for the `error_class` field. LanceDB
|
|
84
|
+
* surfaces schema mismatches, IO errors, and lock contention with
|
|
85
|
+
* different shapes — pick a coarse class so dashboards can group.
|
|
86
|
+
*/
|
|
87
|
+
function classifyError(err) {
|
|
88
|
+
const msg = err instanceof Error ? err.message : String(err);
|
|
89
|
+
const lower = msg.toLowerCase();
|
|
90
|
+
if (lower.includes("schema"))
|
|
91
|
+
return "schema_mismatch";
|
|
92
|
+
if (lower.includes("lock") || lower.includes("conflict"))
|
|
93
|
+
return "lock_conflict";
|
|
94
|
+
if (lower.includes("enoent") || lower.includes("eacces") || lower.includes("eio")) {
|
|
95
|
+
return "io";
|
|
96
|
+
}
|
|
97
|
+
if (err instanceof TypeError)
|
|
98
|
+
return "type";
|
|
99
|
+
return "unknown";
|
|
100
|
+
}
|
|
101
|
+
/**
|
|
102
|
+
* Tolerant reader. One bad JSON line is skipped; the rest are returned.
|
|
103
|
+
* Returns [] if the file is missing.
|
|
104
|
+
*/
|
|
105
|
+
export function readIndexEvents(dataDir) {
|
|
106
|
+
let raw;
|
|
107
|
+
try {
|
|
108
|
+
raw = fs.readFileSync(indexEventsFilePath(dataDir), "utf-8");
|
|
109
|
+
}
|
|
110
|
+
catch {
|
|
111
|
+
return [];
|
|
112
|
+
}
|
|
113
|
+
const out = [];
|
|
114
|
+
for (const line of raw.split("\n")) {
|
|
115
|
+
if (!line)
|
|
116
|
+
continue;
|
|
117
|
+
let parsed;
|
|
118
|
+
try {
|
|
119
|
+
parsed = JSON.parse(line);
|
|
120
|
+
}
|
|
121
|
+
catch {
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
if (!isIndexEvent(parsed))
|
|
125
|
+
continue;
|
|
126
|
+
out.push(parsed);
|
|
127
|
+
}
|
|
128
|
+
return out;
|
|
129
|
+
}
|
|
130
|
+
function isIndexEvent(value) {
|
|
131
|
+
if (value === null || typeof value !== "object")
|
|
132
|
+
return false;
|
|
133
|
+
const v = value;
|
|
134
|
+
if (!(typeof v["ts"] === "string" &&
|
|
135
|
+
typeof v["table"] === "string" &&
|
|
136
|
+
typeof v["pipeline"] === "string" &&
|
|
137
|
+
typeof v["operation"] === "string" &&
|
|
138
|
+
(v["outcome"] === "success" || v["outcome"] === "error") &&
|
|
139
|
+
typeof v["row_count"] === "number" &&
|
|
140
|
+
typeof v["ms"] === "number")) {
|
|
141
|
+
return false;
|
|
142
|
+
}
|
|
143
|
+
const phase = v["phase"];
|
|
144
|
+
if (phase !== undefined && phase !== "live" && phase !== "shadow")
|
|
145
|
+
return false;
|
|
146
|
+
return true;
|
|
147
|
+
}
|
|
148
|
+
//# sourceMappingURL=index-events.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index-events.js","sourceRoot":"","sources":["../../../src/observability/index-events.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;GAgBG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AA2C7B,MAAM,qBAAqB,GAAG,GAAG,CAAC;AAWlC,MAAM,UAAU,mBAAmB,CAAC,OAAe;IACjD,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,eAAe,EAAE,oBAAoB,CAAC,CAAC;AACnE,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gBAAgB,CAC9B,OAAe,EACf,KAAiB;IAEjB,MAAM,QAAQ,GAAG,mBAAmB,CAAC,OAAO,CAAC,CAAC;IAC9C,IAAI,CAAC;QACH,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC1D,EAAE,CAAC,cAAc,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,GAAG,IAAI,CAAC,CAAC;IAC5D,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,OAAO,CAAC,MAAM,CAAC,KAAK,CAClB,gCAAgC,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CACrF,CAAC;IACJ,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,gBAAgB,CACpC,OAAe,EACf,GAAsB,EACtB,EAAoB;IAEpB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACzB,MAAM,KAAK,GAAG,GAAG,CAAC,KAAK,IAAI,MAAM,CAAC;IAClC,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,EAAE,EAAE,CAAC;QAC1B,gBAAgB,CAAC,OAAO,EAAE;YACxB,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC5B,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,OAAO,EAAE,SAAS;YAClB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YACtB,KAAK;SACN,CAAC,CAAC;QACH,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;QACjE,gBAAgB,CAAC,OAAO,EAAE;YACxB,EAAE,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;YAC5B,KAAK,EAAE,GAAG,CAAC,KAAK;YAChB,QAAQ,EAAE,GAAG,CAAC,QAAQ;YACtB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,OAAO,EAAE,OAAO;YAChB,SAAS,EAAE,GAAG,CAAC,SAAS;YACxB,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK;YACtB,KAAK;YACL,WAAW,EAAE,aAAa,CAAC,GAAG,CAAC;YAC/B,aAAa,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,qBAAqB,CAAC;SACvD,CAAC,CAAC;QACH,MAAM,GAAG,CAAC;IACZ,CAAC;AACH,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CAAC,GAAY;IACjC,MAAM,GAAG,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC;IAC7D,MAAM,KAAK,GAAG,GAAG,CAAC,WAAW,EAAE,CAAC;IAChC,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,iBAAiB,CAAC;IACvD,IAAI,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC;QAAE,OAAO,eAAe,CAAC;IACjF,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAC,EAAE,CAAC;QAClF,OAAO,IAAI,CAAC;IACd,CAAC;IACD,IAAI,GAAG,YAAY,SAAS;QAAE,OAAO,MAAM,CAAC;IAC5C,OAAO,SAAS,CAAC;AACnB,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAAC,OAAe;IAC7C,IAAI,GAAW,CAAC;IAChB,IAAI,CAAC;QACH,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,mBAAmB,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;IAC/D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;IACD,MAAM,GAAG,GAAiB,EAAE,CAAC;IAC7B,KAAK,MAAM,IAAI,IAAI,GAAG,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QACnC,IAAI,CAAC,IAAI;YAAE,SAAS;QACpB,IAAI,MAAe,CAAC;QACpB,IAAI,CAAC;YACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QAC5B,CAAC;QAAC,MAAM,CAAC;YACP,SAAS;QACX,CAAC;QACD,IAAI,CAAC,YAAY,CAAC,MAAM,CAAC;YAAE,SAAS;QACpC,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IACnB,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC9D,MAAM,CAAC,GAAG,KAAgC,CAAC;IAC3C,IACE,CAAC,CACC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,QAAQ;QAC3B,OAAO,CAAC,CAAC,OAAO,CAAC,KAAK,QAAQ;QAC9B,OAAO,CAAC,CAAC,UAAU,CAAC,KAAK,QAAQ;QACjC,OAAO,CAAC,CAAC,WAAW,CAAC,KAAK,QAAQ;QAClC,CAAC,CAAC,CAAC,SAAS,CAAC,KAAK,SAAS,IAAI,CAAC,CAAC,SAAS,CAAC,KAAK,OAAO,CAAC;QACxD,OAAO,CAAC,CAAC,WAAW,CAAC,KAAK,QAAQ;QAClC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,QAAQ,CAC5B,EACD,CAAC;QACD,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC;IACzB,IAAI,KAAK,KAAK,SAAS,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,QAAQ;QAAE,OAAO,KAAK,CAAC;IAChF,OAAO,IAAI,CAAC;AACd,CAAC"}
|
|
@@ -6,7 +6,8 @@ export interface OllamaEmbedOptions {
|
|
|
6
6
|
baseUrl?: string;
|
|
7
7
|
}
|
|
8
8
|
export declare class OllamaEmbeddingProvider implements EmbeddingProvider {
|
|
9
|
-
|
|
9
|
+
readonly name = "ollama";
|
|
10
|
+
readonly model: string;
|
|
10
11
|
private readonly retry;
|
|
11
12
|
private readonly cachePath;
|
|
12
13
|
private readonly baseUrl;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ollama-embed.d.ts","sourceRoot":"","sources":["../../../src/providers/ollama-embed.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAKjE,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,uBAAwB,YAAW,iBAAiB;IAC/D,
|
|
1
|
+
{"version":3,"file":"ollama-embed.d.ts","sourceRoot":"","sources":["../../../src/providers/ollama-embed.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAKjE,MAAM,WAAW,kBAAkB;IACjC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC7B,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,qBAAa,uBAAwB,YAAW,iBAAiB;IAC/D,QAAQ,CAAC,IAAI,YAAY;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqB;IAC/C,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;gBAErB,IAAI,GAAE,kBAAuB;IAWnC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAWtC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;YAwBxC,OAAO;IAgDrB,OAAO,CAAC,kBAAkB;IAU1B,OAAO,CAAC,QAAQ;IAOhB,OAAO,CAAC,SAAS;IAejB,OAAO,CAAC,UAAU;CASnB"}
|
|
@@ -11,6 +11,7 @@ import crypto from "node:crypto";
|
|
|
11
11
|
import { DEFAULT_RETRY_CONFIG, LOCAL_EMBEDDING_DIMENSIONS } from "./types.js";
|
|
12
12
|
const DEFAULT_BASE_URL = "http://localhost:11434";
|
|
13
13
|
export class OllamaEmbeddingProvider {
|
|
14
|
+
name = "ollama";
|
|
14
15
|
model;
|
|
15
16
|
retry;
|
|
16
17
|
cachePath;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"ollama-embed.js","sourceRoot":"","sources":["../../../src/providers/ollama-embed.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,OAAO,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,YAAY,CAAC;AAE9E,MAAM,gBAAgB,GAAG,wBAAwB,CAAC;AASlD,MAAM,OAAO,uBAAuB;
|
|
1
|
+
{"version":3,"file":"ollama-embed.js","sourceRoot":"","sources":["../../../src/providers/ollama-embed.ts"],"names":[],"mappings":"AAAA;;;;;;GAMG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,OAAO,EAAE,oBAAoB,EAAE,0BAA0B,EAAE,MAAM,YAAY,CAAC;AAE9E,MAAM,gBAAgB,GAAG,wBAAwB,CAAC;AASlD,MAAM,OAAO,uBAAuB;IACzB,IAAI,GAAG,QAAQ,CAAC;IAChB,KAAK,CAAS;IACN,KAAK,CAAc;IACnB,SAAS,CAAqB;IAC9B,OAAO,CAAS;IAEjC,YAAY,OAA2B,EAAE;QACvC,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,kBAAkB,CAAC;QAC9C,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,oBAAoB,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACxD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAChC,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,IAAI,gBAAgB,CAAC;QAEhD,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAE1B,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,4CAA4C,CAAC,CAAC;QAC3E,IAAI,CAAC,kBAAkB,CAAC,MAAM,CAAC,CAAC;QAChC,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC9B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,MAAM,OAAO,GAAwB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QACzE,MAAM,MAAM,GAAsC,EAAE,CAAC;QAErD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YACjE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;gBACxB,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAE,CAAC;gBACjC,IAAI,CAAC,kBAAkB,CAAC,SAAS,CAAC,CAAC;gBACnC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC;gBAChC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,OAAO,OAAqB,CAAC;IAC/B,CAAC;IAEO,KAAK,CAAC,OAAO,CAAC,KAAe;QACnC,iEAAiE;QACjE,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAE,CAAC,CAAC,CAAC,KAAK,CAAC;QACrD,IAAI,SAAkB,CAAC;QAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YAClE,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACpB,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,EAC3C,IAAI,CAAC,KAAK,CAAC,UAAU,CACtB,CAAC;gBACF,MAAM,IAAI,OAAO,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC;YACjD,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,YAAY,EAAE;oBACnD,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;oBAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC;iBACnD,CAAC,CAAC;gBAEH,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;oBAC5C,SAAS,GAAG,IAAI,KAAK,CACnB,oBAAoB,GAAG,CAAC,MAAM,KAAK,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CACtD,CAAC;oBACF,SAAS;gBACX,CAAC;gBAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;oBACZ,MAAM,IAAI,KAAK,CACb,oBAAoB,GAAG,CAAC,MAAM,KAAK,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CACtD,CAAC;gBACJ,CAAC;gBAED,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA+B,CAAC;gBAC9D,OAAO,IAAI,CAAC,UAAU,CAAC;YACzB,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,IAAI,GAAG,YAAY,SAAS,EAAE,CAAC;oBAC7B,SAAS,GAAG,GAAG,CAAC;oBAChB,SAAS;gBACX,CAAC;gBACD,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QAED,MAAM,SAAS,CAAC;IAClB,CAAC;IAEO,kBAAkB,CAAC,SAAmB;QAC5C,IAAI,SAAS,CAAC,MAAM,KAAK,0BAA0B,EAAE,CAAC;YACpD,MAAM,IAAI,KAAK,CACb,iBAAiB,IAAI,CAAC,KAAK,cAAc,SAAS,CAAC,MAAM,6BAA6B,0BAA0B,+BAA+B,0BAA0B,eAAe,CACzL,CAAC;QACJ,CAAC;IACH,CAAC;IAED,+DAA+D;IAEvD,QAAQ,CAAC,IAAY;QAC3B,OAAO,MAAM;aACV,UAAU,CAAC,QAAQ,CAAC;aACpB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;aAC/B,MAAM,CAAC,KAAK,CAAC,CAAC;IACnB,CAAC;IAEO,SAAS,CAAC,IAAY;QAC5B,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;QACtE,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAa,CAAC;YAC3C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,0BAA0B,EAAE,CAAC;gBAC1E,OAAO,MAAM,CAAC;YAChB,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,IAAY,EAAE,SAAmB;QAClD,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;QACtE,IAAI,CAAC;YACH,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QACpD,CAAC;QAAC,MAAM,CAAC;YACP,oCAAoC;QACtC,CAAC;IACH,CAAC;CACF"}
|
|
@@ -1,5 +1,7 @@
|
|
|
1
1
|
import type { EmbeddingProvider, DigestSummarizer, MetadataExtractor, ConversationEntry, ConversationMetadata, DigestContent, RetryConfig } from "./types.js";
|
|
2
2
|
export declare class StubEmbeddingProvider implements EmbeddingProvider {
|
|
3
|
+
readonly name = "stub";
|
|
4
|
+
readonly model = "stub";
|
|
3
5
|
private readonly dimensions;
|
|
4
6
|
constructor(dimensions?: number);
|
|
5
7
|
embed(text: string): Promise<number[]>;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stub.d.ts","sourceRoot":"","sources":["../../../src/providers/stub.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAG9J,qBAAa,qBAAsB,YAAW,iBAAiB;IAC7D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;gBAExB,UAAU,GAAE,MAA6B;IAI/C,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAItC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAGvD;AAED,qBAAa,oBAAqB,YAAW,gBAAgB;IAC3D,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAwB;IAE7C,SAAS,CAAC,aAAa,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC;CAU5E;AAED,qBAAa,qBAAsB,YAAW,iBAAiB;IAC7D,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAwB;IAEnD,oEAAoE;IAC9D,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;CAmBnE"}
|
|
1
|
+
{"version":3,"file":"stub.d.ts","sourceRoot":"","sources":["../../../src/providers/stub.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,iBAAiB,EAAE,gBAAgB,EAAE,iBAAiB,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAG9J,qBAAa,qBAAsB,YAAW,iBAAiB;IAC7D,QAAQ,CAAC,IAAI,UAAU;IACvB,QAAQ,CAAC,KAAK,UAAU;IACxB,OAAO,CAAC,QAAQ,CAAC,UAAU,CAAS;gBAExB,UAAU,GAAE,MAA6B;IAI/C,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAItC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;CAGvD;AAED,qBAAa,oBAAqB,YAAW,gBAAgB;IAC3D,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAwB;IAE7C,SAAS,CAAC,aAAa,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC;CAU5E;AAED,qBAAa,qBAAsB,YAAW,iBAAiB;IAC7D,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAwB;IAEnD,oEAAoE;IAC9D,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC;CAmBnE"}
|
|
@@ -9,6 +9,8 @@
|
|
|
9
9
|
import crypto from "node:crypto";
|
|
10
10
|
import { EMBEDDING_DIMENSIONS, DEFAULT_RETRY_CONFIG } from "./types.js";
|
|
11
11
|
export class StubEmbeddingProvider {
|
|
12
|
+
name = "stub";
|
|
13
|
+
model = "stub";
|
|
12
14
|
dimensions;
|
|
13
15
|
constructor(dimensions = EMBEDDING_DIMENSIONS) {
|
|
14
16
|
this.dimensions = dimensions;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"stub.js","sourceRoot":"","sources":["../../../src/providers/stub.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAExE,MAAM,OAAO,qBAAqB;
|
|
1
|
+
{"version":3,"file":"stub.js","sourceRoot":"","sources":["../../../src/providers/stub.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AACH,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAExE,MAAM,OAAO,qBAAqB;IACvB,IAAI,GAAG,MAAM,CAAC;IACd,KAAK,GAAG,MAAM,CAAC;IACP,UAAU,CAAS;IAEpC,YAAY,aAAqB,oBAAoB;QACnD,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,OAAO,YAAY,CAAC,IAAI,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC;IAC7C,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,YAAY,CAAC,CAAC,EAAE,IAAI,CAAC,UAAU,CAAC,CAAC,CAAC;IAC5D,CAAC;CACF;AAED,MAAM,OAAO,oBAAoB;IACtB,KAAK,GAAgB,oBAAoB,CAAC;IAEnD,KAAK,CAAC,SAAS,CAAC,aAAkC;QAChD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAU,CAAC;QACjC,KAAK,MAAM,CAAC,IAAI,aAAa,EAAE,CAAC;YAC9B,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM;gBAAE,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;QAC1C,CAAC;QACD,OAAO;YACL,OAAO,EAAE,aAAa,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC;YACxD,MAAM,EAAE,CAAC,GAAG,MAAM,CAAC;SACpB,CAAC;IACJ,CAAC;CACF;AAED,MAAM,OAAO,qBAAqB;IACvB,KAAK,GAAgB,oBAAoB,CAAC;IAEnD,oEAAoE;IACpE,KAAK,CAAC,eAAe,CAAC,IAAY;QAChC,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;QAC/D,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC;QAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC;QAC/B,MAAM,UAAU,GAAG,IAAI,CAAC,CAAC,CAAE,GAAG,CAAC,CAAC;QAEhC,MAAM,OAAO,GAAG,CAAC,UAAU,EAAE,MAAM,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,CAAU,CAAC;QACrG,MAAM,OAAO,GAAG,CAAC,UAAU,EAAE,UAAU,EAAE,YAAY,EAAE,OAAO,EAAE,OAAO,EAAE,cAAc,CAAU,CAAC;QAClG,MAAM,SAAS,GAAG,CAAC,MAAM,EAAE,QAAQ,EAAE,KAAK,CAAU,CAAC;QAErD,OAAO;YACL,MAAM,EAAE,OAAO,CAAC,SAAS,CAAE;YAC3B,MAAM,EAAE,OAAO,CAAC,SAAS,CAAE;YAC3B,OAAO,EAAE,SAAS,CAAC,UAAU,CAAE;YAC/B,MAAM,EAAE,CAAC,YAAY,CAAC;YACtB,SAAS,EAAE,EAAE;YACb,WAAW,EAAE,EAAE;SAChB,CAAC;IACJ,CAAC;CACF;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,IAAY,EAAE,UAAkB;IACpD,MAAM,IAAI,GAAG,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,CAAC;IAC/D,MAAM,MAAM,GAAa,EAAE,CAAC;IAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,UAAU,EAAE,CAAC,EAAE,EAAE,CAAC;QACpC,iDAAiD;QACjD,MAAM,IAAI,GAAG,IAAI,CAAC,CAAC,GAAG,IAAI,CAAC,MAAM,CAAE,CAAC;QACpC,MAAM,CAAC,IAAI,CAAC,CAAC,IAAI,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC;IAClC,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
|
@@ -32,6 +32,17 @@ export type SourceScope = (typeof SOURCE_SCOPES)[number];
|
|
|
32
32
|
export declare const EMBEDDING_DIMENSIONS = 512;
|
|
33
33
|
export declare const LOCAL_EMBEDDING_DIMENSIONS = 768;
|
|
34
34
|
export interface EmbeddingProvider {
|
|
35
|
+
/**
|
|
36
|
+
* Provider name (e.g. "voyage", "ollama", "stub"). Used to label
|
|
37
|
+
* embedding-events.jsonl entries without each call site needing to know
|
|
38
|
+
* which provider it's holding.
|
|
39
|
+
*/
|
|
40
|
+
readonly name: string;
|
|
41
|
+
/**
|
|
42
|
+
* Model identifier (e.g. "voyage-3-lite", "nomic-embed-text"). Same
|
|
43
|
+
* purpose as `name` — telemetry labelling, not runtime behavior.
|
|
44
|
+
*/
|
|
45
|
+
readonly model: string;
|
|
35
46
|
/** Generate an embedding vector for a single text. */
|
|
36
47
|
embed(text: string): Promise<number[]>;
|
|
37
48
|
/** Generate embedding vectors for a batch of texts. */
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/providers/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,MAAM,WAAW,WAAW;IAC1B,qEAAqE;IACrE,UAAU,EAAE,MAAM,CAAC;IACnB,gDAAgD;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,oBAAoB,EAAE,WAIlC,CAAC;AAIF,eAAO,MAAM,oBAAoB,4GAQvB,CAAC;AACX,MAAM,MAAM,kBAAkB,GAAG,CAAC,OAAO,oBAAoB,CAAC,CAAC,MAAM,CAAC,CAAC;AAEvE,eAAO,MAAM,OAAO,sFAQV,CAAC;AACX,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9C,eAAO,MAAM,OAAO,mFAOV,CAAC;AACX,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9C,eAAO,MAAM,cAAc,oCAAqC,CAAC;AACjE,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAElE,eAAO,MAAM,YAAY,8DAGf,CAAC;AACX,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC;AAEvD,eAAO,MAAM,aAAa,4CAA6C,CAAC;AACxE,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC;AAIzD,eAAO,MAAM,oBAAoB,MAAM,CAAC;AACxC,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAE9C,MAAM,WAAW,iBAAiB;IAChC,sDAAsD;IACtD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAEvC,uDAAuD;IACvD,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;CAClD;AAID,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,mBAAmB,CAAC;IAC7B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAE5B,mEAAmE;IACnE,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAE7D,yEAAyE;IACzE,aAAa,CAAC,IAAI,4BAA4B,GAAG,SAAS,CAAC;CAC5D;AAID,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,mBAAmB,CAAC;IAC7B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAE5B,wDAAwD;IACxD,SAAS,CAAC,aAAa,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;CACvE"}
|
|
1
|
+
{"version":3,"file":"types.d.ts","sourceRoot":"","sources":["../../../src/providers/types.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,MAAM,WAAW,WAAW;IAC1B,qEAAqE;IACrE,UAAU,EAAE,MAAM,CAAC;IACnB,gDAAgD;IAChD,WAAW,EAAE,MAAM,CAAC;IACpB,+BAA+B;IAC/B,UAAU,EAAE,MAAM,CAAC;CACpB;AAED,eAAO,MAAM,oBAAoB,EAAE,WAIlC,CAAC;AAIF,eAAO,MAAM,oBAAoB,4GAQvB,CAAC;AACX,MAAM,MAAM,kBAAkB,GAAG,CAAC,OAAO,oBAAoB,CAAC,CAAC,MAAM,CAAC,CAAC;AAEvE,eAAO,MAAM,OAAO,sFAQV,CAAC;AACX,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9C,eAAO,MAAM,OAAO,mFAOV,CAAC;AACX,MAAM,MAAM,MAAM,GAAG,CAAC,OAAO,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC;AAE9C,eAAO,MAAM,cAAc,oCAAqC,CAAC;AACjE,MAAM,MAAM,mBAAmB,GAAG,CAAC,OAAO,cAAc,CAAC,CAAC,MAAM,CAAC,CAAC;AAElE,eAAO,MAAM,YAAY,8DAGf,CAAC;AACX,MAAM,MAAM,UAAU,GAAG,CAAC,OAAO,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC;AAEvD,eAAO,MAAM,aAAa,4CAA6C,CAAC;AACxE,MAAM,MAAM,WAAW,GAAG,CAAC,OAAO,aAAa,CAAC,CAAC,MAAM,CAAC,CAAC;AAIzD,eAAO,MAAM,oBAAoB,MAAM,CAAC;AACxC,eAAO,MAAM,0BAA0B,MAAM,CAAC;AAE9C,MAAM,WAAW,iBAAiB;IAChC;;;;OAIG;IACH,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IAEtB;;;OAGG;IACH,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IAEvB,sDAAsD;IACtD,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC;IAEvC,uDAAuD;IACvD,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;CAClD;AAID,MAAM,WAAW,oBAAoB;IACnC,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,mBAAmB,CAAC;IAC7B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,4BAA4B;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,KAAK,EAAE,MAAM,CAAC;IACd,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,CAAC,EAAE,MAAM,CAAC;CAClB;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAE5B,mEAAmE;IACnE,eAAe,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAE7D,yEAAyE;IACzE,aAAa,CAAC,IAAI,4BAA4B,GAAG,SAAS,CAAC;CAC5D;AAID,MAAM,WAAW,iBAAiB;IAChC,EAAE,EAAE,MAAM,CAAC;IACX,OAAO,EAAE,MAAM,CAAC;IAChB,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,kBAAkB,CAAC;IAC3B,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,mBAAmB,CAAC;IAC7B,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,SAAS,EAAE,MAAM,EAAE,CAAC;IACpB,WAAW,EAAE,MAAM,EAAE,CAAC;CACvB;AAED,MAAM,WAAW,aAAa;IAC5B,OAAO,EAAE,MAAM,CAAC;IAChB,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,KAAK,EAAE,WAAW,CAAC;IAE5B,wDAAwD;IACxD,SAAS,CAAC,aAAa,EAAE,iBAAiB,EAAE,GAAG,OAAO,CAAC,aAAa,CAAC,CAAC;CACvE"}
|
|
@@ -7,8 +7,9 @@ export interface VoyageProviderOptions {
|
|
|
7
7
|
cachePath?: string;
|
|
8
8
|
}
|
|
9
9
|
export declare class VoyageEmbeddingProvider implements EmbeddingProvider {
|
|
10
|
+
readonly name = "voyage";
|
|
10
11
|
private readonly apiKey;
|
|
11
|
-
|
|
12
|
+
readonly model: string;
|
|
12
13
|
private readonly retry;
|
|
13
14
|
private readonly cachePath;
|
|
14
15
|
constructor(opts: VoyageProviderOptions);
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"voyage.d.ts","sourceRoot":"","sources":["../../../src/providers/voyage.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAKjE,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC7B,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,uBAAwB,YAAW,iBAAiB;IAC/D,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,
|
|
1
|
+
{"version":3,"file":"voyage.d.ts","sourceRoot":"","sources":["../../../src/providers/voyage.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,iBAAiB,EAAE,WAAW,EAAE,MAAM,YAAY,CAAC;AAKjE,MAAM,WAAW,qBAAqB;IACpC,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,KAAK,CAAC,EAAE,OAAO,CAAC,WAAW,CAAC,CAAC;IAC7B,+DAA+D;IAC/D,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB;AAED,qBAAa,uBAAwB,YAAW,iBAAiB;IAC/D,QAAQ,CAAC,IAAI,YAAY;IACzB,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAS;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAc;IACpC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAqB;gBAEnC,IAAI,EAAE,qBAAqB;IAWjC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC;IAUtC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC;IAwBtD,wDAAwD;YAC1C,OAAO;IAsDrB,OAAO,CAAC,QAAQ;IAOhB,OAAO,CAAC,SAAS;IAejB,OAAO,CAAC,UAAU;CASnB"}
|
|
@@ -10,6 +10,7 @@ import crypto from "node:crypto";
|
|
|
10
10
|
import { DEFAULT_RETRY_CONFIG, EMBEDDING_DIMENSIONS } from "./types.js";
|
|
11
11
|
const VOYAGE_API_URL = "https://api.voyageai.com/v1/embeddings";
|
|
12
12
|
export class VoyageEmbeddingProvider {
|
|
13
|
+
name = "voyage";
|
|
13
14
|
apiKey;
|
|
14
15
|
model;
|
|
15
16
|
retry;
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"voyage.js","sourceRoot":"","sources":["../../../src/providers/voyage.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAExE,MAAM,cAAc,GAAG,wCAAwC,CAAC;AAUhE,MAAM,OAAO,uBAAuB;
|
|
1
|
+
{"version":3,"file":"voyage.js","sourceRoot":"","sources":["../../../src/providers/voyage.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AACH,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,MAAM,MAAM,aAAa,CAAC;AAEjC,OAAO,EAAE,oBAAoB,EAAE,oBAAoB,EAAE,MAAM,YAAY,CAAC;AAExE,MAAM,cAAc,GAAG,wCAAwC,CAAC;AAUhE,MAAM,OAAO,uBAAuB;IACzB,IAAI,GAAG,QAAQ,CAAC;IACR,MAAM,CAAS;IACvB,KAAK,CAAS;IACN,KAAK,CAAc;IACnB,SAAS,CAAqB;IAE/C,YAAY,IAA2B;QACrC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,MAAM,CAAC;QAC1B,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,eAAe,CAAC;QAC3C,IAAI,CAAC,KAAK,GAAG,EAAE,GAAG,oBAAoB,EAAE,GAAG,IAAI,CAAC,KAAK,EAAE,CAAC;QACxD,IAAI,CAAC,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC;QAEhC,IAAI,IAAI,CAAC,SAAS,EAAE,CAAC;YACnB,EAAE,CAAC,SAAS,CAAC,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,MAAM,GAAG,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,MAAM;YAAE,OAAO,MAAM,CAAC;QAE1B,MAAM,CAAC,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;QAC5C,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QACrE,IAAI,CAAC,UAAU,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;QAC9B,OAAO,MAAM,CAAC;IAChB,CAAC;IAED,KAAK,CAAC,UAAU,CAAC,KAAe;QAC9B,mDAAmD;QACnD,MAAM,OAAO,GAAwB,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC;QACzE,MAAM,MAAM,GAAsC,EAAE,CAAC;QAErD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;YACtC,IAAI,OAAO,CAAC,CAAC,CAAC,KAAK,IAAI,EAAE,CAAC;gBACxB,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,CAAC,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC,CAAE,EAAE,CAAC,CAAC;YAC7C,CAAC;QACH,CAAC;QAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACtB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC;YACjE,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,MAAM,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACvC,MAAM,IAAI,GAAG,MAAM,CAAC,CAAC,CAAE,CAAC;gBACxB,MAAM,SAAS,GAAG,UAAU,CAAC,CAAC,CAAE,CAAC;gBACjC,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,SAAS,CAAC;gBAChC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,IAAI,EAAE,SAAS,CAAC,CAAC;YACxC,CAAC;QACH,CAAC;QAED,OAAO,OAAqB,CAAC;IAC/B,CAAC;IAED,wDAAwD;IAChD,KAAK,CAAC,OAAO,CAAC,KAAe;QACnC,IAAI,SAAkB,CAAC;QAEvB,KAAK,IAAI,OAAO,GAAG,CAAC,EAAE,OAAO,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,EAAE,OAAO,EAAE,EAAE,CAAC;YAClE,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;gBAChB,MAAM,KAAK,GAAG,IAAI,CAAC,GAAG,CACpB,IAAI,CAAC,KAAK,CAAC,WAAW,GAAG,CAAC,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,EAC3C,IAAI,CAAC,KAAK,CAAC,UAAU,CACtB,CAAC;gBACF,MAAM,KAAK,CAAC,KAAK,CAAC,CAAC;YACrB,CAAC;YAED,IAAI,CAAC;gBACH,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,cAAc,EAAE;oBACtC,MAAM,EAAE,MAAM;oBACd,OAAO,EAAE;wBACP,cAAc,EAAE,kBAAkB;wBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;qBACvC;oBACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC;wBACnB,KAAK,EAAE,KAAK;wBACZ,KAAK,EAAE,IAAI,CAAC,KAAK;qBAClB,CAAC;iBACH,CAAC,CAAC;gBAEH,IAAI,GAAG,CAAC,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,MAAM,IAAI,GAAG,EAAE,CAAC;oBAC5C,SAAS,GAAG,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,MAAM,KAAK,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;oBACvE,SAAS,CAAC,oBAAoB;gBAChC,CAAC;gBAED,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;oBACZ,MAAM,IAAI,KAAK,CAAC,cAAc,GAAG,CAAC,MAAM,KAAK,MAAM,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;gBACnE,CAAC;gBAED,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAE7B,CAAC;gBAEF,OAAO,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;YAC3C,CAAC;YAAC,OAAO,GAAY,EAAE,CAAC;gBACtB,IAAI,GAAG,YAAY,SAAS,EAAE,CAAC;oBAC7B,mCAAmC;oBACnC,SAAS,GAAG,GAAG,CAAC;oBAChB,SAAS;gBACX,CAAC;gBACD,MAAM,GAAG,CAAC;YACZ,CAAC;QACH,CAAC;QAED,MAAM,SAAS,CAAC;IAClB,CAAC;IAED,qBAAqB;IAEb,QAAQ,CAAC,IAAY;QAC3B,OAAO,MAAM;aACV,UAAU,CAAC,QAAQ,CAAC;aACpB,MAAM,CAAC,GAAG,IAAI,CAAC,KAAK,IAAI,IAAI,EAAE,CAAC;aAC/B,MAAM,CAAC,KAAK,CAAC,CAAC;IACnB,CAAC;IAEO,SAAS,CAAC,IAAY;QAC5B,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO,IAAI,CAAC;QACjC,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;QACtE,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,EAAE,CAAC,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;YAC3C,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,GAAG,CAAa,CAAC;YAC3C,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,KAAK,oBAAoB,EAAE,CAAC;gBACpE,OAAO,MAAM,CAAC;YAChB,CAAC;YACD,OAAO,IAAI,CAAC;QACd,CAAC;QAAC,MAAM,CAAC;YACP,OAAO,IAAI,CAAC;QACd,CAAC;IACH,CAAC;IAEO,UAAU,CAAC,IAAY,EAAE,SAAmB;QAClD,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,OAAO;QAC5B,MAAM,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,SAAS,EAAE,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC;QACtE,IAAI,CAAC;YACH,EAAE,CAAC,aAAa,CAAC,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,SAAS,CAAC,CAAC,CAAC;QACpD,CAAC;QAAC,MAAM,CAAC;YACP,oCAAoC;QACtC,CAAC;IACH,CAAC;CACF;AAED,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC"}
|