@chunktrace/sdk 0.0.0-stage → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +43 -2
- package/dist/adapters.d.ts +37 -0
- package/dist/adapters.d.ts.map +1 -0
- package/dist/adapters.js +2 -0
- package/dist/adapters.js.map +1 -0
- package/dist/index.d.ts +11 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +9 -0
- package/dist/index.js.map +1 -0
- package/dist/memory.d.ts +25 -0
- package/dist/memory.d.ts.map +1 -0
- package/dist/memory.js +83 -0
- package/dist/memory.js.map +1 -0
- package/dist/pgvector.d.ts +51 -0
- package/dist/pgvector.d.ts.map +1 -0
- package/dist/pgvector.js +87 -0
- package/dist/pgvector.js.map +1 -0
- package/dist/profiler.d.ts +87 -0
- package/dist/profiler.d.ts.map +1 -0
- package/dist/profiler.js +205 -0
- package/dist/profiler.js.map +1 -0
- package/dist/sinks.d.ts +12 -0
- package/dist/sinks.d.ts.map +1 -0
- package/dist/sinks.js +35 -0
- package/dist/sinks.js.map +1 -0
- package/package.json +29 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Hathor Software, LLC
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,44 @@
|
|
|
1
|
-
#
|
|
1
|
+
# @chunktrace/sdk
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
Instrumentation for [ChunkTrace](https://chunktrace.com), which diagnoses RAG retrieval failures from a single query trace. Wrap your retrieval pipeline with the profiler; each query produces one trace, and the diagnosis names the stage that lost the evidence the answer needed: the index, the metadata filter, the similarity threshold, `top_k`, the reranker or the context budget. No LLM calls are made.
|
|
4
|
+
|
|
5
|
+
```
|
|
6
|
+
npm install @chunktrace/sdk
|
|
7
|
+
```
|
|
8
|
+
|
|
9
|
+
```ts
|
|
10
|
+
import { createProfiler, httpSink, pgvectorStore } from "@chunktrace/sdk";
|
|
11
|
+
|
|
12
|
+
const profiler = createProfiler({
|
|
13
|
+
app: { name: "support-bot" },
|
|
14
|
+
sink: httpSink({ endpoint: "http://localhost:8787", apiKey: process.env.CHUNKTRACE_KEY! }),
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
const store = pgvectorStore({ name: "docs", db: pgPool, table: "chunks", embed });
|
|
18
|
+
|
|
19
|
+
const run = profiler.trace({
|
|
20
|
+
query: userQuestion,
|
|
21
|
+
label: { type: "keyword", values: ["30 days"] }, // what the answer should have cited
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
let hits = await run.retrieve(store, {
|
|
25
|
+
top_k: 5,
|
|
26
|
+
similarity_threshold: 0.75,
|
|
27
|
+
metadata_filter: { tenant: "acme" },
|
|
28
|
+
diagnose: true, // one extra unfiltered query so removals become visible
|
|
29
|
+
});
|
|
30
|
+
hits = await run.rerank(myReranker, hits, { top_n: 3 });
|
|
31
|
+
const { text } = run.buildContext(hits, { token_budget: 2000 });
|
|
32
|
+
const answer = await llm(text, userQuestion);
|
|
33
|
+
await run.finish({ answer, answer_correct: false });
|
|
34
|
+
|
|
35
|
+
console.log(run.diagnose().findings.find((f) => f.primary)?.message);
|
|
36
|
+
// "Chunk refund-policy#2 ranked 6 among eligible results; top_k=5 cut it."
|
|
37
|
+
```
|
|
38
|
+
|
|
39
|
+
- `run.diagnose()` works locally, with no server. `httpSink` also sends the trace to a ChunkTrace server, where it shows up in the dashboard; `fileSink` writes traces to disk and `multiSink` combines sinks.
|
|
40
|
+
- `diagnose: true` is meant for a failing query or a sample of traffic, not every request: it costs one extra vector search and no inference.
|
|
41
|
+
- Privacy defaults: chunk text is never sent unless `includeText: true` (label matching runs in your process and ships only `matches_label`), and only metadata keys referenced by the filter are sent unless `metadataKeys` says otherwise.
|
|
42
|
+
- Vector stores: `pgvectorStore` and `memoryStore` are included; anything else implements the `VectorStore` interface.
|
|
43
|
+
|
|
44
|
+
Requires Node.js 22 or later. MIT licensed. Copyright © 2026 Hathor Software, LLC.
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import type { MetadataFilter, MetadataValue } from "@chunktrace/core";
|
|
2
|
+
/** A chunk as returned by a vector store. The SDK turns these into trace Candidates. */
|
|
3
|
+
export type StoreHit = {
|
|
4
|
+
chunk_id: string;
|
|
5
|
+
document_id?: string;
|
|
6
|
+
score: number;
|
|
7
|
+
metadata?: Record<string, MetadataValue>;
|
|
8
|
+
text?: string;
|
|
9
|
+
token_count?: number;
|
|
10
|
+
};
|
|
11
|
+
export type SearchRequest = {
|
|
12
|
+
queryText: string;
|
|
13
|
+
k: number;
|
|
14
|
+
filter?: MetadataFilter;
|
|
15
|
+
/** Drop hits with score < threshold before applying k. */
|
|
16
|
+
threshold?: number;
|
|
17
|
+
};
|
|
18
|
+
/**
|
|
19
|
+
* The one interface an integration must implement. Results come back in rank order.
|
|
20
|
+
* The adapter is responsible for applying filter, threshold and k the same way the
|
|
21
|
+
* production pipeline does, so the trace reflects what the app actually saw.
|
|
22
|
+
*/
|
|
23
|
+
export interface VectorStore {
|
|
24
|
+
readonly name: string;
|
|
25
|
+
readonly provider?: string;
|
|
26
|
+
search(req: SearchRequest): Promise<StoreHit[]>;
|
|
27
|
+
}
|
|
28
|
+
export interface Reranker {
|
|
29
|
+
readonly model?: string;
|
|
30
|
+
/** Returns a score per input chunk id. Higher = more relevant. */
|
|
31
|
+
score(queryText: string, hits: StoreHit[]): Promise<Array<{
|
|
32
|
+
chunk_id: string;
|
|
33
|
+
score: number;
|
|
34
|
+
}>>;
|
|
35
|
+
}
|
|
36
|
+
export type Embedder = (text: string) => Promise<number[]> | number[];
|
|
37
|
+
//# sourceMappingURL=adapters.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"adapters.d.ts","sourceRoot":"","sources":["../src/adapters.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AAEtE,wFAAwF;AACxF,MAAM,MAAM,QAAQ,GAAG;IACrB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IACzC,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAC1B,SAAS,EAAE,MAAM,CAAC;IAClB,CAAC,EAAE,MAAM,CAAC;IACV,MAAM,CAAC,EAAE,cAAc,CAAC;IACxB,0DAA0D;IAC1D,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAEF;;;;GAIG;AACH,MAAM,WAAW,WAAW;IAC1B,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,CAAC,EAAE,MAAM,CAAC;IAC3B,MAAM,CAAC,GAAG,EAAE,aAAa,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAC;CACjD;AAED,MAAM,WAAW,QAAQ;IACvB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,kEAAkE;IAClE,KAAK,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,EAAE,QAAQ,EAAE,GAAG,OAAO,CAAC,KAAK,CAAC;QAAE,QAAQ,EAAE,MAAM,CAAC;QAAC,KAAK,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC,CAAC;CACjG;AAED,MAAM,MAAM,QAAQ,GAAG,CAAC,IAAI,EAAE,MAAM,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,GAAG,MAAM,EAAE,CAAC"}
|
package/dist/adapters.js
ADDED
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"adapters.js","sourceRoot":"","sources":["../src/adapters.ts"],"names":[],"mappings":""}
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export { createProfiler, Run, matchesLabel } from "./profiler.js";
|
|
2
|
+
export type { ProfilerOptions, RetrieveOptions, RerankOptions, ContextOptions, TraceSink } from "./profiler.js";
|
|
3
|
+
export type { VectorStore, Reranker, Embedder, StoreHit, SearchRequest } from "./adapters.js";
|
|
4
|
+
export { fileSink, httpSink, multiSink } from "./sinks.js";
|
|
5
|
+
export { memoryStore, hashedBagOfWords } from "./memory.js";
|
|
6
|
+
export type { MemoryChunk } from "./memory.js";
|
|
7
|
+
export { pgvectorStore, pgvectorSchemaSql, buildSearchSql } from "./pgvector.js";
|
|
8
|
+
export type { Queryable, PgvectorOptions } from "./pgvector.js";
|
|
9
|
+
export { diagnose, formatReport, validateTrace, ENGINE_VERSION } from "@chunktrace/core";
|
|
10
|
+
export type * from "@chunktrace/core";
|
|
11
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,cAAc,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAClE,YAAY,EAAE,eAAe,EAAE,eAAe,EAAE,aAAa,EAAE,cAAc,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAChH,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,QAAQ,EAAE,QAAQ,EAAE,aAAa,EAAE,MAAM,eAAe,CAAC;AAC9F,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAG3D,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAC5D,YAAY,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAC/C,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AACjF,YAAY,EAAE,SAAS,EAAE,eAAe,EAAE,MAAM,eAAe,CAAC;AAGhE,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACzF,mBAAmB,kBAAkB,CAAC"}
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
// SDK
|
|
2
|
+
export { createProfiler, Run, matchesLabel } from "./profiler.js";
|
|
3
|
+
export { fileSink, httpSink, multiSink } from "./sinks.js";
|
|
4
|
+
// Adapters
|
|
5
|
+
export { memoryStore, hashedBagOfWords } from "./memory.js";
|
|
6
|
+
export { pgvectorStore, pgvectorSchemaSql, buildSearchSql } from "./pgvector.js";
|
|
7
|
+
// Re-exported from core so an app only needs one import.
|
|
8
|
+
export { diagnose, formatReport, validateTrace, ENGINE_VERSION } from "@chunktrace/core";
|
|
9
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,MAAM;AACN,OAAO,EAAE,cAAc,EAAE,GAAG,EAAE,YAAY,EAAE,MAAM,eAAe,CAAC;AAGlE,OAAO,EAAE,QAAQ,EAAE,QAAQ,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAE3D,WAAW;AACX,OAAO,EAAE,WAAW,EAAE,gBAAgB,EAAE,MAAM,aAAa,CAAC;AAE5D,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAE,cAAc,EAAE,MAAM,eAAe,CAAC;AAGjF,yDAAyD;AACzD,OAAO,EAAE,QAAQ,EAAE,YAAY,EAAE,aAAa,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC"}
|
package/dist/memory.d.ts
ADDED
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import type { MetadataValue } from "@chunktrace/core";
|
|
2
|
+
import type { Embedder, VectorStore } from "./adapters.js";
|
|
3
|
+
export type MemoryChunk = {
|
|
4
|
+
chunk_id: string;
|
|
5
|
+
document_id?: string;
|
|
6
|
+
text: string;
|
|
7
|
+
metadata?: Record<string, MetadataValue>;
|
|
8
|
+
};
|
|
9
|
+
/**
|
|
10
|
+
* In-memory cosine-similarity store. Used by the sample app and tests; also fine for
|
|
11
|
+
* small corpora in real apps. Applies filter → threshold → k, in that order.
|
|
12
|
+
*/
|
|
13
|
+
export declare function memoryStore(opts: {
|
|
14
|
+
name: string;
|
|
15
|
+
embed: Embedder;
|
|
16
|
+
chunks: MemoryChunk[];
|
|
17
|
+
}): VectorStore & {
|
|
18
|
+
add(chunks: MemoryChunk[]): Promise<void>;
|
|
19
|
+
};
|
|
20
|
+
/**
|
|
21
|
+
* Deterministic hashed bag-of-words embedding. No model, no API call. Good enough for
|
|
22
|
+
* demos and tests where you need plausible, stable similarity scores.
|
|
23
|
+
*/
|
|
24
|
+
export declare function hashedBagOfWords(dim?: number): Embedder;
|
|
25
|
+
//# sourceMappingURL=memory.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.d.ts","sourceRoot":"","sources":["../src/memory.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,kBAAkB,CAAC;AACtD,OAAO,KAAK,EAAE,QAAQ,EAA2B,WAAW,EAAE,MAAM,eAAe,CAAC;AAEpF,MAAM,MAAM,WAAW,GAAG;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;CAC1C,CAAC;AAEF;;;GAGG;AACH,wBAAgB,WAAW,CAAC,IAAI,EAAE;IAAE,IAAI,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,QAAQ,CAAC;IAAC,MAAM,EAAE,WAAW,EAAE,CAAA;CAAE,GAAG,WAAW,GAAG;IACzG,GAAG,CAAC,MAAM,EAAE,WAAW,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CAC3C,CA+BA;AAcD;;;GAGG;AACH,wBAAgB,gBAAgB,CAAC,GAAG,SAAM,GAAG,QAAQ,CASpD"}
|
package/dist/memory.js
ADDED
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
import { evaluateFilter } from "@chunktrace/core";
|
|
2
|
+
/**
|
|
3
|
+
* In-memory cosine-similarity store. Used by the sample app and tests; also fine for
|
|
4
|
+
* small corpora in real apps. Applies filter → threshold → k, in that order.
|
|
5
|
+
*/
|
|
6
|
+
export function memoryStore(opts) {
|
|
7
|
+
const rows = [];
|
|
8
|
+
const ready = (async () => {
|
|
9
|
+
for (const c of opts.chunks)
|
|
10
|
+
rows.push({ ...c, vec: await opts.embed(c.text) });
|
|
11
|
+
})();
|
|
12
|
+
return {
|
|
13
|
+
name: opts.name,
|
|
14
|
+
provider: "memory",
|
|
15
|
+
async add(chunks) {
|
|
16
|
+
await ready;
|
|
17
|
+
for (const c of chunks)
|
|
18
|
+
rows.push({ ...c, vec: await opts.embed(c.text) });
|
|
19
|
+
},
|
|
20
|
+
async search(req) {
|
|
21
|
+
await ready;
|
|
22
|
+
const q = await opts.embed(req.queryText);
|
|
23
|
+
return rows
|
|
24
|
+
.filter((r) => evaluateFilter(req.filter, r.metadata).passes)
|
|
25
|
+
.map((r) => ({ r, score: round(cosine(q, r.vec)) }))
|
|
26
|
+
.filter((x) => req.threshold === undefined || x.score >= req.threshold)
|
|
27
|
+
.sort((a, b) => b.score - a.score || a.r.chunk_id.localeCompare(b.r.chunk_id))
|
|
28
|
+
.slice(0, req.k)
|
|
29
|
+
.map(({ r, score }) => ({
|
|
30
|
+
chunk_id: r.chunk_id,
|
|
31
|
+
...(r.document_id ? { document_id: r.document_id } : {}),
|
|
32
|
+
score,
|
|
33
|
+
...(r.metadata ? { metadata: r.metadata } : {}),
|
|
34
|
+
text: r.text,
|
|
35
|
+
}));
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
function cosine(a, b) {
|
|
40
|
+
let dot = 0, na = 0, nb = 0;
|
|
41
|
+
for (let i = 0; i < a.length; i++) {
|
|
42
|
+
dot += a[i] * b[i];
|
|
43
|
+
na += a[i] * a[i];
|
|
44
|
+
nb += b[i] * b[i];
|
|
45
|
+
}
|
|
46
|
+
return na && nb ? dot / Math.sqrt(na * nb) : 0;
|
|
47
|
+
}
|
|
48
|
+
const round = (x) => Math.round(x * 1000) / 1000;
|
|
49
|
+
/**
|
|
50
|
+
* Deterministic hashed bag-of-words embedding. No model, no API call. Good enough for
|
|
51
|
+
* demos and tests where you need plausible, stable similarity scores.
|
|
52
|
+
*/
|
|
53
|
+
export function hashedBagOfWords(dim = 256) {
|
|
54
|
+
return (text) => {
|
|
55
|
+
const v = new Array(dim).fill(0);
|
|
56
|
+
for (const tok of tokenize(text)) {
|
|
57
|
+
const h = fnv1a(tok);
|
|
58
|
+
v[h % dim] += (h & 1) === 0 ? 1 : -1;
|
|
59
|
+
}
|
|
60
|
+
return v;
|
|
61
|
+
};
|
|
62
|
+
}
|
|
63
|
+
const STOP = new Set(["the", "a", "an", "is", "are", "of", "to", "in", "and", "or", "my", "i", "what", "how", "do", "can", "be", "for", "it", "your", "from", "within", "with"]);
|
|
64
|
+
function tokenize(text) {
|
|
65
|
+
return text
|
|
66
|
+
.toLowerCase()
|
|
67
|
+
.replace(/[^a-z0-9\s]/g, " ")
|
|
68
|
+
.split(/\s+/)
|
|
69
|
+
.filter((t) => t && !STOP.has(t))
|
|
70
|
+
.map(stem);
|
|
71
|
+
}
|
|
72
|
+
function stem(t) {
|
|
73
|
+
return t.replace(/(ing|ed|es|s)$/, "");
|
|
74
|
+
}
|
|
75
|
+
function fnv1a(s) {
|
|
76
|
+
let h = 0x811c9dc5;
|
|
77
|
+
for (let i = 0; i < s.length; i++) {
|
|
78
|
+
h ^= s.charCodeAt(i);
|
|
79
|
+
h = Math.imul(h, 0x01000193) >>> 0;
|
|
80
|
+
}
|
|
81
|
+
return h;
|
|
82
|
+
}
|
|
83
|
+
//# sourceMappingURL=memory.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"memory.js","sourceRoot":"","sources":["../src/memory.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAWlD;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,IAA8D;IAGxF,MAAM,IAAI,GAA2C,EAAE,CAAC;IACxD,MAAM,KAAK,GAAG,CAAC,KAAK,IAAI,EAAE;QACxB,KAAK,MAAM,CAAC,IAAI,IAAI,CAAC,MAAM;YAAE,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAClF,CAAC,CAAC,EAAE,CAAC;IAEL,OAAO;QACL,IAAI,EAAE,IAAI,CAAC,IAAI;QACf,QAAQ,EAAE,QAAQ;QAClB,KAAK,CAAC,GAAG,CAAC,MAAM;YACd,MAAM,KAAK,CAAC;YACZ,KAAK,MAAM,CAAC,IAAI,MAAM;gBAAE,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,CAAC,EAAE,GAAG,EAAE,MAAM,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QAC7E,CAAC;QACD,KAAK,CAAC,MAAM,CAAC,GAAkB;YAC7B,MAAM,KAAK,CAAC;YACZ,MAAM,CAAC,GAAG,MAAM,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YAC1C,OAAO,IAAI;iBACR,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC;iBAC5D,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,CAAC,CAAC,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;iBACnD,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,GAAG,CAAC,SAAS,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,IAAI,GAAG,CAAC,SAAS,CAAC;iBACtE,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,aAAa,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;iBAC7E,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;iBACf,GAAG,CAAC,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC,CAAC;gBACtB,QAAQ,EAAE,CAAC,CAAC,QAAQ;gBACpB,GAAG,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxD,KAAK;gBACL,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/C,IAAI,EAAE,CAAC,CAAC,IAAI;aACb,CAAC,CAAC,CAAC;QACR,CAAC;KACF,CAAC;AACJ,CAAC;AAED,SAAS,MAAM,CAAC,CAAW,EAAE,CAAW;IACtC,IAAI,GAAG,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,EAAE,EAAE,GAAG,CAAC,CAAC;IAC5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,GAAG,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QACnB,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;QAClB,EAAE,IAAI,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC;IACpB,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,GAAG,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;AACjD,CAAC;AAED,MAAM,KAAK,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;AAEzD;;;GAGG;AACH,MAAM,UAAU,gBAAgB,CAAC,GAAG,GAAG,GAAG;IACxC,OAAO,CAAC,IAAY,EAAE,EAAE;QACtB,MAAM,CAAC,GAAG,IAAI,KAAK,CAAS,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACzC,KAAK,MAAM,GAAG,IAAI,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;YACrB,CAAC,CAAC,CAAC,GAAG,GAAG,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACvC,CAAC;QACD,OAAO,CAAC,CAAC;IACX,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,IAAI,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,GAAG,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC;AAEjL,SAAS,QAAQ,CAAC,IAAY;IAC5B,OAAO,IAAI;SACR,WAAW,EAAE;SACb,OAAO,CAAC,cAAc,EAAE,GAAG,CAAC;SAC5B,KAAK,CAAC,KAAK,CAAC;SACZ,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;SAChC,GAAG,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,SAAS,IAAI,CAAC,CAAS;IACrB,OAAO,CAAC,CAAC,OAAO,CAAC,gBAAgB,EAAE,EAAE,CAAC,CAAC;AACzC,CAAC;AAED,SAAS,KAAK,CAAC,CAAS;IACtB,IAAI,CAAC,GAAG,UAAU,CAAC;IACnB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAClC,CAAC,IAAI,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QACrB,CAAC,GAAG,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IACrC,CAAC;IACD,OAAO,CAAC,CAAC;AACX,CAAC"}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { MetadataFilter } from "@chunktrace/core";
|
|
2
|
+
import type { Embedder, SearchRequest, VectorStore } from "./adapters.js";
|
|
3
|
+
/** Minimal shape of `pg`'s Pool/Client.query so we don't depend on the package. */
|
|
4
|
+
export type Queryable = {
|
|
5
|
+
query(sql: string, params: unknown[]): Promise<{
|
|
6
|
+
rows: Record<string, unknown>[];
|
|
7
|
+
}>;
|
|
8
|
+
};
|
|
9
|
+
export type PgvectorOptions = {
|
|
10
|
+
name: string;
|
|
11
|
+
db: Queryable;
|
|
12
|
+
embed: Embedder;
|
|
13
|
+
table: string;
|
|
14
|
+
/** Column names. Defaults match the schema in `pgvectorSchemaSql()`. */
|
|
15
|
+
columns?: Partial<{
|
|
16
|
+
id: string;
|
|
17
|
+
document_id: string;
|
|
18
|
+
embedding: string;
|
|
19
|
+
metadata: string;
|
|
20
|
+
text: string;
|
|
21
|
+
}>;
|
|
22
|
+
/** Whether to return chunk text to the app (the profiler still only ships it if includeText is set). */
|
|
23
|
+
selectText?: boolean;
|
|
24
|
+
};
|
|
25
|
+
declare const DEFAULT_COLUMNS: {
|
|
26
|
+
id: string;
|
|
27
|
+
document_id: string;
|
|
28
|
+
embedding: string;
|
|
29
|
+
metadata: string;
|
|
30
|
+
text: string;
|
|
31
|
+
};
|
|
32
|
+
/**
|
|
33
|
+
* pgvector adapter. Cosine similarity = 1 - (embedding <=> query).
|
|
34
|
+
* Filter and threshold are applied in SQL before LIMIT, matching the rule engine's
|
|
35
|
+
* filter → threshold → top_k ordering.
|
|
36
|
+
*/
|
|
37
|
+
export declare function pgvectorStore(o: PgvectorOptions): VectorStore;
|
|
38
|
+
export declare function buildSearchSql(a: {
|
|
39
|
+
table: string;
|
|
40
|
+
col: typeof DEFAULT_COLUMNS;
|
|
41
|
+
req: SearchRequest;
|
|
42
|
+
vector: number[];
|
|
43
|
+
selectText: boolean;
|
|
44
|
+
}): {
|
|
45
|
+
sql: string;
|
|
46
|
+
params: unknown[];
|
|
47
|
+
};
|
|
48
|
+
/** Reference table DDL for apps that don't already have one. */
|
|
49
|
+
export declare function pgvectorSchemaSql(table?: string, dim?: number): string;
|
|
50
|
+
export type { MetadataFilter };
|
|
51
|
+
//# sourceMappingURL=pgvector.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pgvector.d.ts","sourceRoot":"","sources":["../src/pgvector.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAgB,cAAc,EAAE,MAAM,kBAAkB,CAAC;AACrE,OAAO,KAAK,EAAE,QAAQ,EAAE,aAAa,EAAY,WAAW,EAAE,MAAM,eAAe,CAAC;AAEpF,mFAAmF;AACnF,MAAM,MAAM,SAAS,GAAG;IACtB,KAAK,CAAC,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,GAAG,OAAO,CAAC;QAAE,IAAI,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,CAAA;KAAE,CAAC,CAAC;CACrF,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,EAAE,EAAE,SAAS,CAAC;IACd,KAAK,EAAE,QAAQ,CAAC;IAChB,KAAK,EAAE,MAAM,CAAC;IACd,wEAAwE;IACxE,OAAO,CAAC,EAAE,OAAO,CAAC;QAAE,EAAE,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE,CAAC,CAAC;IAC1G,wGAAwG;IACxG,UAAU,CAAC,EAAE,OAAO,CAAC;CACtB,CAAC;AAEF,QAAA,MAAM,eAAe;;;;;;CAAuG,CAAC;AAE7H;;;;GAIG;AACH,wBAAgB,aAAa,CAAC,CAAC,EAAE,eAAe,GAAG,WAAW,CAkB7D;AAED,wBAAgB,cAAc,CAAC,CAAC,EAAE;IAChC,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,OAAO,eAAe,CAAC;IAC5B,GAAG,EAAE,aAAa,CAAC;IACnB,MAAM,EAAE,MAAM,EAAE,CAAC;IACjB,UAAU,EAAE,OAAO,CAAC;CACrB,GAAG;IAAE,GAAG,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,OAAO,EAAE,CAAA;CAAE,CAsBrC;AA+BD,gEAAgE;AAChE,wBAAgB,iBAAiB,CAAC,KAAK,SAAW,EAAE,GAAG,SAAO,GAAG,MAAM,CAYtE;AAED,YAAY,EAAE,cAAc,EAAE,CAAC"}
|
package/dist/pgvector.js
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
const DEFAULT_COLUMNS = { id: "id", document_id: "document_id", embedding: "embedding", metadata: "metadata", text: "text" };
|
|
2
|
+
/**
|
|
3
|
+
* pgvector adapter. Cosine similarity = 1 - (embedding <=> query).
|
|
4
|
+
* Filter and threshold are applied in SQL before LIMIT, matching the rule engine's
|
|
5
|
+
* filter → threshold → top_k ordering.
|
|
6
|
+
*/
|
|
7
|
+
export function pgvectorStore(o) {
|
|
8
|
+
const col = { ...DEFAULT_COLUMNS, ...(o.columns ?? {}) };
|
|
9
|
+
return {
|
|
10
|
+
name: o.name,
|
|
11
|
+
provider: "pgvector",
|
|
12
|
+
async search(req) {
|
|
13
|
+
const vec = await o.embed(req.queryText);
|
|
14
|
+
const { sql, params } = buildSearchSql({ table: o.table, col, req, vector: vec, selectText: o.selectText ?? true });
|
|
15
|
+
const { rows } = await o.db.query(sql, params);
|
|
16
|
+
return rows.map((r) => ({
|
|
17
|
+
chunk_id: String(r.chunk_id),
|
|
18
|
+
...(r.document_id != null ? { document_id: String(r.document_id) } : {}),
|
|
19
|
+
score: Number(r.score),
|
|
20
|
+
...(r.metadata != null ? { metadata: r.metadata } : {}),
|
|
21
|
+
...(r.text != null ? { text: String(r.text) } : {}),
|
|
22
|
+
}));
|
|
23
|
+
},
|
|
24
|
+
};
|
|
25
|
+
}
|
|
26
|
+
export function buildSearchSql(a) {
|
|
27
|
+
const params = [`[${a.vector.join(",")}]`];
|
|
28
|
+
const where = [];
|
|
29
|
+
const { col } = a;
|
|
30
|
+
for (const [key, clause] of Object.entries(a.req.filter ?? {})) {
|
|
31
|
+
where.push(clauseSql(col.metadata, key, clause, params));
|
|
32
|
+
}
|
|
33
|
+
const scoreExpr = `1 - (${q(col.embedding)} <=> $1::vector)`;
|
|
34
|
+
if (a.req.threshold !== undefined) {
|
|
35
|
+
params.push(a.req.threshold);
|
|
36
|
+
where.push(`${scoreExpr} >= $${params.length}`);
|
|
37
|
+
}
|
|
38
|
+
params.push(a.req.k);
|
|
39
|
+
const sql = `SELECT ${q(col.id)} AS chunk_id, ${q(col.document_id)} AS document_id, ${q(col.metadata)} AS metadata, ` +
|
|
40
|
+
`${a.selectText ? `${q(col.text)} AS text, ` : ""}${scoreExpr} AS score ` +
|
|
41
|
+
`FROM ${q(a.table)}` +
|
|
42
|
+
(where.length ? ` WHERE ${where.join(" AND ")}` : "") +
|
|
43
|
+
` ORDER BY ${q(col.embedding)} <=> $1::vector LIMIT $${params.length}`;
|
|
44
|
+
return { sql, params };
|
|
45
|
+
}
|
|
46
|
+
function clauseSql(mdCol, key, clause, params) {
|
|
47
|
+
const textExpr = `${q(mdCol)}->>${lit(key)}`;
|
|
48
|
+
const numExpr = `(${textExpr})::numeric`;
|
|
49
|
+
if (clause === null || typeof clause !== "object") {
|
|
50
|
+
params.push(String(clause));
|
|
51
|
+
return `${textExpr} = $${params.length}`;
|
|
52
|
+
}
|
|
53
|
+
if ("$in" in clause) {
|
|
54
|
+
params.push(clause.$in.map(String));
|
|
55
|
+
return `${textExpr} = ANY($${params.length}::text[])`;
|
|
56
|
+
}
|
|
57
|
+
if ("$ne" in clause) {
|
|
58
|
+
params.push(String(clause.$ne));
|
|
59
|
+
return `(${textExpr} IS NULL OR ${textExpr} <> $${params.length})`;
|
|
60
|
+
}
|
|
61
|
+
const parts = [];
|
|
62
|
+
for (const [op, sym] of [["$gt", ">"], ["$gte", ">="], ["$lt", "<"], ["$lte", "<="]]) {
|
|
63
|
+
const v = clause[op];
|
|
64
|
+
if (v !== undefined) {
|
|
65
|
+
params.push(v);
|
|
66
|
+
parts.push(`${numExpr} ${sym} $${params.length}`);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
return parts.length ? `(${parts.join(" AND ")})` : "TRUE";
|
|
70
|
+
}
|
|
71
|
+
const q = (ident) => `"${ident.replace(/"/g, '""')}"`;
|
|
72
|
+
const lit = (s) => `'${s.replace(/'/g, "''")}'`;
|
|
73
|
+
/** Reference table DDL for apps that don't already have one. */
|
|
74
|
+
export function pgvectorSchemaSql(table = "chunks", dim = 1536) {
|
|
75
|
+
return [
|
|
76
|
+
`CREATE EXTENSION IF NOT EXISTS vector;`,
|
|
77
|
+
`CREATE TABLE IF NOT EXISTS ${q(table)} (`,
|
|
78
|
+
` id text PRIMARY KEY,`,
|
|
79
|
+
` document_id text,`,
|
|
80
|
+
` text text NOT NULL,`,
|
|
81
|
+
` metadata jsonb NOT NULL DEFAULT '{}',`,
|
|
82
|
+
` embedding vector(${dim}) NOT NULL`,
|
|
83
|
+
`);`,
|
|
84
|
+
`CREATE INDEX IF NOT EXISTS ${q(table + "_embedding_idx")} ON ${q(table)} USING hnsw (embedding vector_cosine_ops);`,
|
|
85
|
+
].join("\n");
|
|
86
|
+
}
|
|
87
|
+
//# sourceMappingURL=pgvector.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pgvector.js","sourceRoot":"","sources":["../src/pgvector.ts"],"names":[],"mappings":"AAmBA,MAAM,eAAe,GAAG,EAAE,EAAE,EAAE,IAAI,EAAE,WAAW,EAAE,aAAa,EAAE,SAAS,EAAE,WAAW,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;AAE7H;;;;GAIG;AACH,MAAM,UAAU,aAAa,CAAC,CAAkB;IAC9C,MAAM,GAAG,GAAG,EAAE,GAAG,eAAe,EAAE,GAAG,CAAC,CAAC,CAAC,OAAO,IAAI,EAAE,CAAC,EAAE,CAAC;IACzD,OAAO;QACL,IAAI,EAAE,CAAC,CAAC,IAAI;QACZ,QAAQ,EAAE,UAAU;QACpB,KAAK,CAAC,MAAM,CAAC,GAAkB;YAC7B,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;YACzC,MAAM,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,cAAc,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,CAAC,UAAU,IAAI,IAAI,EAAE,CAAC,CAAC;YACpH,MAAM,EAAE,IAAI,EAAE,GAAG,MAAM,CAAC,CAAC,EAAE,CAAC,KAAK,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YAC/C,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;gBACtB,QAAQ,EAAE,MAAM,CAAC,CAAC,CAAC,QAAQ,CAAC;gBAC5B,GAAG,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,MAAM,CAAC,CAAC,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACxE,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC,KAAK,CAAC;gBACtB,GAAG,CAAC,CAAC,CAAC,QAAQ,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAgC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC/E,GAAG,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACpD,CAAC,CAAC,CAAC;QACN,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,cAAc,CAAC,CAM9B;IACC,MAAM,MAAM,GAAc,CAAC,IAAI,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;IACtD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;IAElB,KAAK,MAAM,CAAC,GAAG,EAAE,MAAM,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,MAAM,IAAI,EAAE,CAAC,EAAE,CAAC;QAC/D,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,QAAQ,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,SAAS,GAAG,QAAQ,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,kBAAkB,CAAC;IAC7D,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,KAAK,SAAS,EAAE,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QAC7B,KAAK,CAAC,IAAI,CAAC,GAAG,SAAS,QAAQ,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;IAClD,CAAC;IACD,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC;IAErB,MAAM,GAAG,GACP,UAAU,CAAC,CAAC,GAAG,CAAC,EAAE,CAAC,iBAAiB,CAAC,CAAC,GAAG,CAAC,WAAW,CAAC,oBAAoB,CAAC,CAAC,GAAG,CAAC,QAAQ,CAAC,gBAAgB;QACzG,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC,CAAC,EAAE,GAAG,SAAS,YAAY;QACzE,QAAQ,CAAC,CAAC,CAAC,CAAC,KAAK,CAAC,EAAE;QACpB,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,UAAU,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACrD,aAAa,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,0BAA0B,MAAM,CAAC,MAAM,EAAE,CAAC;IACzE,OAAO,EAAE,GAAG,EAAE,MAAM,EAAE,CAAC;AACzB,CAAC;AAED,SAAS,SAAS,CAAC,KAAa,EAAE,GAAW,EAAE,MAAoB,EAAE,MAAiB;IACpF,MAAM,QAAQ,GAAG,GAAG,CAAC,CAAC,KAAK,CAAC,MAAM,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;IAC7C,MAAM,OAAO,GAAG,IAAI,QAAQ,YAAY,CAAC;IACzC,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,MAAM,KAAK,QAAQ,EAAE,CAAC;QAClD,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;QAC5B,OAAO,GAAG,QAAQ,OAAO,MAAM,CAAC,MAAM,EAAE,CAAC;IAC3C,CAAC;IACD,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;QACpB,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC;QACpC,OAAO,GAAG,QAAQ,WAAW,MAAM,CAAC,MAAM,WAAW,CAAC;IACxD,CAAC;IACD,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;QACpB,MAAM,CAAC,IAAI,CAAC,MAAM,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC;QAChC,OAAO,IAAI,QAAQ,eAAe,QAAQ,QAAQ,MAAM,CAAC,MAAM,GAAG,CAAC;IACrE,CAAC;IACD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,MAAM,CAAC,EAAE,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,GAAG,CAAC,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,CAAU,EAAE,CAAC;QAC9F,MAAM,CAAC,GAAI,MAA6C,CAAC,EAAE,CAAC,CAAC;QAC7D,IAAI,CAAC,KAAK,SAAS,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,GAAG,OAAO,IAAI,GAAG,KAAK,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC;QACpD,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC;AAC5D,CAAC;AAED,MAAM,CAAC,GAAG,CAAC,KAAa,EAAE,EAAE,CAAC,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;AAC9D,MAAM,GAAG,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;AAExD,gEAAgE;AAChE,MAAM,UAAU,iBAAiB,CAAC,KAAK,GAAG,QAAQ,EAAE,GAAG,GAAG,IAAI;IAC5D,OAAO;QACL,wCAAwC;QACxC,8BAA8B,CAAC,CAAC,KAAK,CAAC,IAAI;QAC1C,wBAAwB;QACxB,qBAAqB;QACrB,uBAAuB;QACvB,yCAAyC;QACzC,sBAAsB,GAAG,YAAY;QACrC,IAAI;QACJ,8BAA8B,CAAC,CAAC,KAAK,GAAG,gBAAgB,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,4CAA4C;KACrH,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC"}
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
import type { DiagnosisReport, Label, MetadataFilter, Trace } from "@chunktrace/core";
|
|
2
|
+
import type { Reranker, StoreHit, VectorStore } from "./adapters.js";
|
|
3
|
+
export type TraceSink = (trace: Trace) => Promise<void> | void;
|
|
4
|
+
export type ProfilerOptions = {
|
|
5
|
+
app?: Trace["app"];
|
|
6
|
+
/** Where finished traces go. Defaults to an in-memory array readable via profiler.traces. */
|
|
7
|
+
sink?: TraceSink;
|
|
8
|
+
/** Send chunk text in traces. Off by default; label matching happens client-side regardless. */
|
|
9
|
+
includeText?: boolean;
|
|
10
|
+
/** Which metadata keys to include on candidates. Default: only keys referenced by the filter. */
|
|
11
|
+
metadataKeys?: string[] | "all";
|
|
12
|
+
countTokens?: (text: string) => number;
|
|
13
|
+
now?: () => Date;
|
|
14
|
+
idGen?: () => string;
|
|
15
|
+
};
|
|
16
|
+
export type RetrieveOptions = {
|
|
17
|
+
top_k: number;
|
|
18
|
+
similarity_threshold?: number;
|
|
19
|
+
metadata_filter?: MetadataFilter;
|
|
20
|
+
/** Fire a second, unfiltered, unthresholded query so the profiler can see what was removed. */
|
|
21
|
+
diagnose?: boolean | {
|
|
22
|
+
shadow_k?: number;
|
|
23
|
+
};
|
|
24
|
+
};
|
|
25
|
+
export type RerankOptions = {
|
|
26
|
+
top_n?: number;
|
|
27
|
+
score_threshold?: number;
|
|
28
|
+
};
|
|
29
|
+
export type ContextOptions = {
|
|
30
|
+
token_budget?: number;
|
|
31
|
+
max_chunks?: number;
|
|
32
|
+
separator?: string;
|
|
33
|
+
};
|
|
34
|
+
export declare function createProfiler(opts?: ProfilerOptions): {
|
|
35
|
+
trace: (init: {
|
|
36
|
+
query: string;
|
|
37
|
+
label?: Label;
|
|
38
|
+
trace_id?: string;
|
|
39
|
+
}) => Run;
|
|
40
|
+
traces: Trace[];
|
|
41
|
+
};
|
|
42
|
+
type ResolvedOptions = ProfilerOptions & {
|
|
43
|
+
sink: TraceSink;
|
|
44
|
+
countTokens: (t: string) => number;
|
|
45
|
+
now: () => Date;
|
|
46
|
+
idGen: () => string;
|
|
47
|
+
};
|
|
48
|
+
/** One query's lifecycle. Call retrieve → (rerank) → buildContext → finish. */
|
|
49
|
+
export declare class Run {
|
|
50
|
+
private readonly init;
|
|
51
|
+
private readonly o;
|
|
52
|
+
readonly trace_id: string;
|
|
53
|
+
private readonly started;
|
|
54
|
+
private retrieval?;
|
|
55
|
+
private rerankStage?;
|
|
56
|
+
private context?;
|
|
57
|
+
private textById;
|
|
58
|
+
private hitById;
|
|
59
|
+
private finished?;
|
|
60
|
+
constructor(init: {
|
|
61
|
+
query: string;
|
|
62
|
+
label?: Label;
|
|
63
|
+
trace_id?: string;
|
|
64
|
+
}, o: ResolvedOptions);
|
|
65
|
+
retrieve(store: VectorStore, r: RetrieveOptions): Promise<StoreHit[]>;
|
|
66
|
+
rerank(reranker: Reranker, hits: StoreHit[], r: RerankOptions): Promise<StoreHit[]>;
|
|
67
|
+
/** Assembles the prompt context, recording exactly which chunks made it in and why others didn't. */
|
|
68
|
+
buildContext(hits: StoreHit[], c?: ContextOptions): {
|
|
69
|
+
hits: StoreHit[];
|
|
70
|
+
text: string;
|
|
71
|
+
};
|
|
72
|
+
/** Marks a chunk as excluded for an application-specific reason (e.g. a custom relevance gate). */
|
|
73
|
+
exclude(chunk_id: string, detail: string): void;
|
|
74
|
+
finish(g?: {
|
|
75
|
+
model?: string;
|
|
76
|
+
answer?: string;
|
|
77
|
+
answer_correct?: boolean;
|
|
78
|
+
}): Promise<Trace>;
|
|
79
|
+
/** Runs the rule engine locally on the finished trace. */
|
|
80
|
+
diagnose(): DiagnosisReport;
|
|
81
|
+
private remember;
|
|
82
|
+
private toCandidate;
|
|
83
|
+
}
|
|
84
|
+
/** Client-side label matching, so chunk text never has to leave the app. */
|
|
85
|
+
export declare function matchesLabel(label: Label | undefined, h: StoreHit, text: string | undefined): boolean | undefined;
|
|
86
|
+
export {};
|
|
87
|
+
//# sourceMappingURL=profiler.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"profiler.d.ts","sourceRoot":"","sources":["../src/profiler.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAa,eAAe,EAAE,KAAK,EAAE,cAAc,EAAE,KAAK,EAAE,MAAM,kBAAkB,CAAC;AACjG,OAAO,KAAK,EAAE,QAAQ,EAAE,QAAQ,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAErE,MAAM,MAAM,SAAS,GAAG,CAAC,KAAK,EAAE,KAAK,KAAK,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAE/D,MAAM,MAAM,eAAe,GAAG;IAC5B,GAAG,CAAC,EAAE,KAAK,CAAC,KAAK,CAAC,CAAC;IACnB,6FAA6F;IAC7F,IAAI,CAAC,EAAE,SAAS,CAAC;IACjB,gGAAgG;IAChG,WAAW,CAAC,EAAE,OAAO,CAAC;IACtB,iGAAiG;IACjG,YAAY,CAAC,EAAE,MAAM,EAAE,GAAG,KAAK,CAAC;IAChC,WAAW,CAAC,EAAE,CAAC,IAAI,EAAE,MAAM,KAAK,MAAM,CAAC;IACvC,GAAG,CAAC,EAAE,MAAM,IAAI,CAAC;IACjB,KAAK,CAAC,EAAE,MAAM,MAAM,CAAC;CACtB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,oBAAoB,CAAC,EAAE,MAAM,CAAC;IAC9B,eAAe,CAAC,EAAE,cAAc,CAAC;IACjC,+FAA+F;IAC/F,QAAQ,CAAC,EAAE,OAAO,GAAG;QAAE,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,CAAC;CAC5C,CAAC;AAEF,MAAM,MAAM,aAAa,GAAG;IAAE,KAAK,CAAC,EAAE,MAAM,CAAC;IAAC,eAAe,CAAC,EAAE,MAAM,CAAA;CAAE,CAAC;AAEzE,MAAM,MAAM,cAAc,GAAG;IAC3B,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,SAAS,CAAC,EAAE,MAAM,CAAC;CACpB,CAAC;AAIF,wBAAgB,cAAc,CAAC,IAAI,GAAE,eAAoB;kBAOlC;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,KAAK,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE;;EAKzE;AAED,KAAK,eAAe,GAAG,eAAe,GAAG;IACvC,IAAI,EAAE,SAAS,CAAC;IAChB,WAAW,EAAE,CAAC,CAAC,EAAE,MAAM,KAAK,MAAM,CAAC;IACnC,GAAG,EAAE,MAAM,IAAI,CAAC;IAChB,KAAK,EAAE,MAAM,MAAM,CAAC;CACrB,CAAC;AAEF,+EAA+E;AAC/E,qBAAa,GAAG;IAWZ,OAAO,CAAC,QAAQ,CAAC,IAAI;IACrB,OAAO,CAAC,QAAQ,CAAC,CAAC;IAXpB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAS;IACjC,OAAO,CAAC,SAAS,CAAC,CAAqB;IACvC,OAAO,CAAC,WAAW,CAAC,CAAkB;IACtC,OAAO,CAAC,OAAO,CAAC,CAAmB;IACnC,OAAO,CAAC,QAAQ,CAA6B;IAC7C,OAAO,CAAC,OAAO,CAA+B;IAC9C,OAAO,CAAC,QAAQ,CAAC,CAAQ;gBAGN,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,KAAK,CAAC,EAAE,KAAK,CAAC;QAAC,QAAQ,CAAC,EAAE,MAAM,CAAA;KAAE,EACzD,CAAC,EAAE,eAAe;IAM/B,QAAQ,CAAC,KAAK,EAAE,WAAW,EAAE,CAAC,EAAE,eAAe,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IA+BrE,MAAM,CAAC,QAAQ,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,EAAE,aAAa,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;IAsBzF,qGAAqG;IACrG,YAAY,CAAC,IAAI,EAAE,QAAQ,EAAE,EAAE,CAAC,GAAE,cAAmB,GAAG;QAAE,IAAI,EAAE,QAAQ,EAAE,CAAC;QAAC,IAAI,EAAE,MAAM,CAAA;KAAE;IAoC1F,mGAAmG;IACnG,OAAO,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM;IAMlC,MAAM,CAAC,CAAC,GAAE;QAAE,KAAK,CAAC,EAAE,MAAM,CAAC;QAAC,MAAM,CAAC,EAAE,MAAM,CAAC;QAAC,cAAc,CAAC,EAAE,OAAO,CAAA;KAAO,GAAG,OAAO,CAAC,KAAK,CAAC;IAmBnG,0DAA0D;IAC1D,QAAQ,IAAI,eAAe;IAO3B,OAAO,CAAC,QAAQ;IAOhB,OAAO,CAAC,WAAW;CAapB;AAgBD,4EAA4E;AAC5E,wBAAgB,YAAY,CAAC,KAAK,EAAE,KAAK,GAAG,SAAS,EAAE,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,GAAG,SAAS,GAAG,OAAO,GAAG,SAAS,CAOjH"}
|
package/dist/profiler.js
ADDED
|
@@ -0,0 +1,205 @@
|
|
|
1
|
+
import { diagnose } from "@chunktrace/core";
|
|
2
|
+
const defaultCountTokens = (t) => Math.ceil(t.length / 4);
|
|
3
|
+
export function createProfiler(opts = {}) {
|
|
4
|
+
const traces = [];
|
|
5
|
+
const sink = opts.sink ?? ((t) => void traces.push(t));
|
|
6
|
+
const countTokens = opts.countTokens ?? defaultCountTokens;
|
|
7
|
+
const now = opts.now ?? (() => new Date());
|
|
8
|
+
const idGen = opts.idGen ?? (() => `tr_${Date.now().toString(36)}_${Math.random().toString(36).slice(2, 8)}`);
|
|
9
|
+
function trace(init) {
|
|
10
|
+
return new Run(init, { ...opts, sink, countTokens, now, idGen });
|
|
11
|
+
}
|
|
12
|
+
return { trace, traces };
|
|
13
|
+
}
|
|
14
|
+
/** One query's lifecycle. Call retrieve → (rerank) → buildContext → finish. */
|
|
15
|
+
export class Run {
|
|
16
|
+
init;
|
|
17
|
+
o;
|
|
18
|
+
trace_id;
|
|
19
|
+
started;
|
|
20
|
+
retrieval;
|
|
21
|
+
rerankStage;
|
|
22
|
+
context;
|
|
23
|
+
textById = new Map();
|
|
24
|
+
hitById = new Map();
|
|
25
|
+
finished;
|
|
26
|
+
constructor(init, o) {
|
|
27
|
+
this.init = init;
|
|
28
|
+
this.o = o;
|
|
29
|
+
this.trace_id = init.trace_id ?? o.idGen();
|
|
30
|
+
this.started = o.now().toISOString();
|
|
31
|
+
}
|
|
32
|
+
async retrieve(store, r) {
|
|
33
|
+
if (this.retrieval)
|
|
34
|
+
throw new Error("retrieve() may only be called once per run (v0: one retrieval per trace)");
|
|
35
|
+
const hits = await store.search({
|
|
36
|
+
queryText: this.init.query,
|
|
37
|
+
k: r.top_k,
|
|
38
|
+
filter: r.metadata_filter,
|
|
39
|
+
threshold: r.similarity_threshold,
|
|
40
|
+
});
|
|
41
|
+
this.remember(hits);
|
|
42
|
+
let shadow;
|
|
43
|
+
if (r.diagnose) {
|
|
44
|
+
const shadow_k = typeof r.diagnose === "object" && r.diagnose.shadow_k ? r.diagnose.shadow_k : Math.max(50, r.top_k * 10);
|
|
45
|
+
const shadowHits = await store.search({ queryText: this.init.query, k: shadow_k });
|
|
46
|
+
this.remember(shadowHits);
|
|
47
|
+
shadow = { top_k: shadow_k, candidates: shadowHits.map((h, i) => this.toCandidate(h, i + 1, r.metadata_filter)) };
|
|
48
|
+
}
|
|
49
|
+
this.retrieval = {
|
|
50
|
+
index: { name: store.name, ...(store.provider ? { provider: store.provider } : {}) },
|
|
51
|
+
params: {
|
|
52
|
+
top_k: r.top_k,
|
|
53
|
+
...(r.similarity_threshold !== undefined ? { similarity_threshold: r.similarity_threshold } : {}),
|
|
54
|
+
...(r.metadata_filter ? { metadata_filter: r.metadata_filter } : {}),
|
|
55
|
+
},
|
|
56
|
+
candidates: hits.map((h, i) => this.toCandidate(h, i + 1, r.metadata_filter)),
|
|
57
|
+
...(shadow ? { shadow } : {}),
|
|
58
|
+
};
|
|
59
|
+
return hits;
|
|
60
|
+
}
|
|
61
|
+
async rerank(reranker, hits, r) {
|
|
62
|
+
if (!this.retrieval)
|
|
63
|
+
throw new Error("call retrieve() before rerank()");
|
|
64
|
+
const scores = await reranker.score(this.init.query, hits);
|
|
65
|
+
const scoreById = new Map(scores.map((s) => [s.chunk_id, s.score]));
|
|
66
|
+
const ordered = [...hits]
|
|
67
|
+
.map((h, i) => ({ h, before: i + 1, score: scoreById.get(h.chunk_id) ?? Number.NEGATIVE_INFINITY }))
|
|
68
|
+
.sort((a, b) => b.score - a.score || a.before - b.before);
|
|
69
|
+
const results = ordered.map((x, i) => {
|
|
70
|
+
const after = i + 1;
|
|
71
|
+
const kept = (r.top_n === undefined || after <= r.top_n) && (r.score_threshold === undefined || x.score >= r.score_threshold);
|
|
72
|
+
return { chunk_id: x.h.chunk_id, rank_before: x.before, rank_after: after, score: x.score, kept };
|
|
73
|
+
});
|
|
74
|
+
this.rerankStage = {
|
|
75
|
+
...(reranker.model ? { model: reranker.model } : {}),
|
|
76
|
+
...(r.top_n !== undefined ? { top_n: r.top_n } : {}),
|
|
77
|
+
...(r.score_threshold !== undefined ? { score_threshold: r.score_threshold } : {}),
|
|
78
|
+
results,
|
|
79
|
+
};
|
|
80
|
+
return ordered.filter((_, i) => results[i].kept).map((x) => x.h);
|
|
81
|
+
}
|
|
82
|
+
/** Assembles the prompt context, recording exactly which chunks made it in and why others didn't. */
|
|
83
|
+
buildContext(hits, c = {}) {
|
|
84
|
+
if (!this.retrieval)
|
|
85
|
+
throw new Error("call retrieve() before buildContext()");
|
|
86
|
+
const included = [];
|
|
87
|
+
const excluded = [];
|
|
88
|
+
const kept = [];
|
|
89
|
+
const seen = new Set();
|
|
90
|
+
let used = 0;
|
|
91
|
+
for (const h of hits) {
|
|
92
|
+
const text = h.text ?? this.textById.get(h.chunk_id) ?? "";
|
|
93
|
+
const t = h.token_count ?? this.o.countTokens(text);
|
|
94
|
+
if (seen.has(h.chunk_id)) {
|
|
95
|
+
excluded.push({ chunk_id: h.chunk_id, reason: "dedupe" });
|
|
96
|
+
continue;
|
|
97
|
+
}
|
|
98
|
+
seen.add(h.chunk_id);
|
|
99
|
+
if (c.max_chunks !== undefined && kept.length >= c.max_chunks) {
|
|
100
|
+
excluded.push({ chunk_id: h.chunk_id, reason: "max_chunks", detail: `max_chunks=${c.max_chunks}` });
|
|
101
|
+
continue;
|
|
102
|
+
}
|
|
103
|
+
if (c.token_budget !== undefined && used + t > c.token_budget) {
|
|
104
|
+
excluded.push({ chunk_id: h.chunk_id, reason: "token_budget", detail: `needed ${t}, ${c.token_budget - used} left` });
|
|
105
|
+
continue;
|
|
106
|
+
}
|
|
107
|
+
used += t;
|
|
108
|
+
kept.push(h);
|
|
109
|
+
included.push({ chunk_id: h.chunk_id, order: included.length + 1, token_count: t });
|
|
110
|
+
}
|
|
111
|
+
this.context = {
|
|
112
|
+
...(c.token_budget !== undefined ? { token_budget: c.token_budget } : {}),
|
|
113
|
+
included,
|
|
114
|
+
excluded,
|
|
115
|
+
};
|
|
116
|
+
const text = kept.map((h) => h.text ?? this.textById.get(h.chunk_id) ?? "").join(c.separator ?? "\n\n");
|
|
117
|
+
return { hits: kept, text };
|
|
118
|
+
}
|
|
119
|
+
/** Marks a chunk as excluded for an application-specific reason (e.g. a custom relevance gate). */
|
|
120
|
+
exclude(chunk_id, detail) {
|
|
121
|
+
if (!this.context)
|
|
122
|
+
throw new Error("call buildContext() before exclude()");
|
|
123
|
+
this.context.excluded.push({ chunk_id, reason: "custom", detail });
|
|
124
|
+
this.context.included = this.context.included.filter((x) => x.chunk_id !== chunk_id);
|
|
125
|
+
}
|
|
126
|
+
async finish(g = {}) {
|
|
127
|
+
if (!this.retrieval)
|
|
128
|
+
throw new Error("call retrieve() before finish()");
|
|
129
|
+
const trace = {
|
|
130
|
+
schema_version: "0",
|
|
131
|
+
trace_id: this.trace_id,
|
|
132
|
+
timestamp: this.started,
|
|
133
|
+
...(this.o.app ? { app: this.o.app } : {}),
|
|
134
|
+
query: { text: this.init.query },
|
|
135
|
+
...(this.init.label ? { label: this.init.label } : {}),
|
|
136
|
+
retrieval: this.retrieval,
|
|
137
|
+
...(this.rerankStage ? { rerank: this.rerankStage } : {}),
|
|
138
|
+
context: this.context ?? { included: [], excluded: [] },
|
|
139
|
+
...(Object.keys(g).length ? { generation: g } : {}),
|
|
140
|
+
};
|
|
141
|
+
this.finished = trace;
|
|
142
|
+
await this.o.sink(trace);
|
|
143
|
+
return trace;
|
|
144
|
+
}
|
|
145
|
+
/** Runs the rule engine locally on the finished trace. */
|
|
146
|
+
diagnose() {
|
|
147
|
+
if (!this.finished)
|
|
148
|
+
throw new Error("call finish() before diagnose()");
|
|
149
|
+
return diagnose(this.finished);
|
|
150
|
+
}
|
|
151
|
+
/* ---------- internals ---------- */
|
|
152
|
+
remember(hits) {
|
|
153
|
+
for (const h of hits) {
|
|
154
|
+
this.hitById.set(h.chunk_id, h);
|
|
155
|
+
if (h.text !== undefined)
|
|
156
|
+
this.textById.set(h.chunk_id, h.text);
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
toCandidate(h, rank, filter) {
|
|
160
|
+
const text = h.text ?? this.textById.get(h.chunk_id);
|
|
161
|
+
const c = { chunk_id: h.chunk_id, rank, score: h.score };
|
|
162
|
+
if (h.document_id)
|
|
163
|
+
c.document_id = h.document_id;
|
|
164
|
+
const tokens = h.token_count ?? (text !== undefined ? this.o.countTokens(text) : undefined);
|
|
165
|
+
if (tokens !== undefined)
|
|
166
|
+
c.token_count = tokens;
|
|
167
|
+
const md = pickMetadata(h.metadata, filter, this.o.metadataKeys);
|
|
168
|
+
if (md)
|
|
169
|
+
c.metadata = md;
|
|
170
|
+
const m = matchesLabel(this.init.label, h, text);
|
|
171
|
+
if (m !== undefined)
|
|
172
|
+
c.matches_label = m;
|
|
173
|
+
if (this.o.includeText && text !== undefined)
|
|
174
|
+
c.text = text;
|
|
175
|
+
return c;
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
function pickMetadata(md, filter, keys) {
|
|
179
|
+
if (!md)
|
|
180
|
+
return undefined;
|
|
181
|
+
if (keys === "all")
|
|
182
|
+
return md;
|
|
183
|
+
const wanted = new Set([...(keys ?? []), ...Object.keys(filter ?? {})]);
|
|
184
|
+
if (wanted.size === 0)
|
|
185
|
+
return undefined;
|
|
186
|
+
const out = {};
|
|
187
|
+
for (const k of wanted)
|
|
188
|
+
if (k in md)
|
|
189
|
+
out[k] = md[k];
|
|
190
|
+
return out;
|
|
191
|
+
}
|
|
192
|
+
/** Client-side label matching, so chunk text never has to leave the app. */
|
|
193
|
+
export function matchesLabel(label, h, text) {
|
|
194
|
+
if (!label)
|
|
195
|
+
return undefined;
|
|
196
|
+
if (label.type === "chunk")
|
|
197
|
+
return label.chunk_ids.includes(h.chunk_id);
|
|
198
|
+
if (label.type === "document")
|
|
199
|
+
return h.document_id !== undefined ? label.document_ids.includes(h.document_id) : undefined;
|
|
200
|
+
if (text === undefined)
|
|
201
|
+
return undefined;
|
|
202
|
+
const t = text.toLowerCase();
|
|
203
|
+
return label.values.some((v) => t.includes(v.toLowerCase()));
|
|
204
|
+
}
|
|
205
|
+
//# sourceMappingURL=profiler.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"profiler.js","sourceRoot":"","sources":["../src/profiler.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAmC5C,MAAM,kBAAkB,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;AAElE,MAAM,UAAU,cAAc,CAAC,OAAwB,EAAE;IACvD,MAAM,MAAM,GAAY,EAAE,CAAC;IAC3B,MAAM,IAAI,GAAc,IAAI,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,CAAC;IAClE,MAAM,WAAW,GAAG,IAAI,CAAC,WAAW,IAAI,kBAAkB,CAAC;IAC3D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC;IAC3C,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC,MAAM,IAAI,CAAC,GAAG,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE,CAAC,CAAC;IAE9G,SAAS,KAAK,CAAC,IAAyD;QACtE,OAAO,IAAI,GAAG,CAAC,IAAI,EAAE,EAAE,GAAG,IAAI,EAAE,IAAI,EAAE,WAAW,EAAE,GAAG,EAAE,KAAK,EAAE,CAAC,CAAC;IACnE,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;AAC3B,CAAC;AASD,+EAA+E;AAC/E,MAAM,OAAO,GAAG;IAWK;IACA;IAXV,QAAQ,CAAS;IACT,OAAO,CAAS;IACzB,SAAS,CAAsB;IAC/B,WAAW,CAAmB;IAC9B,OAAO,CAAoB;IAC3B,QAAQ,GAAG,IAAI,GAAG,EAAkB,CAAC;IACrC,OAAO,GAAG,IAAI,GAAG,EAAoB,CAAC;IACtC,QAAQ,CAAS;IAEzB,YACmB,IAAyD,EACzD,CAAkB;QADlB,SAAI,GAAJ,IAAI,CAAqD;QACzD,MAAC,GAAD,CAAC,CAAiB;QAEnC,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ,IAAI,CAAC,CAAC,KAAK,EAAE,CAAC;QAC3C,IAAI,CAAC,OAAO,GAAG,CAAC,CAAC,GAAG,EAAE,CAAC,WAAW,EAAE,CAAC;IACvC,CAAC;IAED,KAAK,CAAC,QAAQ,CAAC,KAAkB,EAAE,CAAkB;QACnD,IAAI,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,0EAA0E,CAAC,CAAC;QAChH,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC;YAC9B,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK;YAC1B,CAAC,EAAE,CAAC,CAAC,KAAK;YACV,MAAM,EAAE,CAAC,CAAC,eAAe;YACzB,SAAS,EAAE,CAAC,CAAC,oBAAoB;SAClC,CAAC,CAAC;QACH,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QAEpB,IAAI,MAAoC,CAAC;QACzC,IAAI,CAAC,CAAC,QAAQ,EAAE,CAAC;YACf,MAAM,QAAQ,GAAG,OAAO,CAAC,CAAC,QAAQ,KAAK,QAAQ,IAAI,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;YAC1H,MAAM,UAAU,GAAG,MAAM,KAAK,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC;YACnF,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC;YAC1B,MAAM,GAAG,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,EAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC,EAAE,CAAC;QACpH,CAAC;QAED,IAAI,CAAC,SAAS,GAAG;YACf,KAAK,EAAE,EAAE,IAAI,EAAE,KAAK,CAAC,IAAI,EAAE,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE;YACpF,MAAM,EAAE;gBACN,KAAK,EAAE,CAAC,CAAC,KAAK;gBACd,GAAG,CAAC,CAAC,CAAC,oBAAoB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,oBAAoB,EAAE,CAAC,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;gBACjG,GAAG,CAAC,CAAC,CAAC,eAAe,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;aACrE;YACD,UAAU,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,EAAE,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,eAAe,CAAC,CAAC;YAC7E,GAAG,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAC9B,CAAC;QACF,OAAO,IAAI,CAAC;IACd,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,QAAkB,EAAE,IAAgB,EAAE,CAAgB;QACjE,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACxE,MAAM,MAAM,GAAG,MAAM,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC3D,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;QACpE,MAAM,OAAO,GAAG,CAAC,GAAG,IAAI,CAAC;aACtB,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,GAAG,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,MAAM,CAAC,iBAAiB,EAAE,CAAC,CAAC;aACnG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC,MAAM,CAAC,CAAC;QAC5D,MAAM,OAAO,GAAG,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE;YACnC,MAAM,KAAK,GAAG,CAAC,GAAG,CAAC,CAAC;YACpB,MAAM,IAAI,GACR,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,IAAI,KAAK,IAAI,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,eAAe,KAAK,SAAS,IAAI,CAAC,CAAC,KAAK,IAAI,CAAC,CAAC,eAAe,CAAC,CAAC;YACnH,OAAO,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,CAAC,QAAQ,EAAE,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE,UAAU,EAAE,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC;QACpG,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,WAAW,GAAG;YACjB,GAAG,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,GAAG,CAAC,CAAC,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACpD,GAAG,CAAC,CAAC,CAAC,eAAe,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,CAAC,CAAC,eAAe,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAClF,OAAO;SACR,CAAC;QACF,OAAO,OAAO,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACnE,CAAC;IAED,qGAAqG;IACrG,YAAY,CAAC,IAAgB,EAAE,IAAoB,EAAE;QACnD,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,uCAAuC,CAAC,CAAC;QAC9E,MAAM,QAAQ,GAAiC,EAAE,CAAC;QAClD,MAAM,QAAQ,GAAiC,EAAE,CAAC;QAClD,MAAM,IAAI,GAAe,EAAE,CAAC;QAC5B,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;QAC/B,IAAI,IAAI,GAAG,CAAC,CAAC;QACb,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC;YAC3D,MAAM,CAAC,GAAG,CAAC,CAAC,WAAW,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC;YACpD,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,EAAE,CAAC;gBACzB,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,CAAC,CAAC;gBAC1D,SAAS;YACX,CAAC;YACD,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;YACrB,IAAI,CAAC,CAAC,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,UAAU,EAAE,CAAC;gBAC9D,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,YAAY,EAAE,MAAM,EAAE,cAAc,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;gBACpG,SAAS;YACX,CAAC;YACD,IAAI,CAAC,CAAC,YAAY,KAAK,SAAS,IAAI,IAAI,GAAG,CAAC,GAAG,CAAC,CAAC,YAAY,EAAE,CAAC;gBAC9D,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,cAAc,EAAE,MAAM,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC,YAAY,GAAG,IAAI,OAAO,EAAE,CAAC,CAAC;gBACtH,SAAS;YACX,CAAC;YACD,IAAI,IAAI,CAAC,CAAC;YACV,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACb,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,KAAK,EAAE,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,WAAW,EAAE,CAAC,EAAE,CAAC,CAAC;QACtF,CAAC;QACD,IAAI,CAAC,OAAO,GAAG;YACb,GAAG,CAAC,CAAC,CAAC,YAAY,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,YAAY,EAAE,CAAC,CAAC,YAAY,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzE,QAAQ;YACR,QAAQ;SACT,CAAC;QACF,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,IAAI,MAAM,CAAC,CAAC;QACxG,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC;IAC9B,CAAC;IAED,mGAAmG;IACnG,OAAO,CAAC,QAAgB,EAAE,MAAc;QACtC,IAAI,CAAC,IAAI,CAAC,OAAO;YAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;QAC3E,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,EAAE,MAAM,EAAE,QAAQ,EAAE,MAAM,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,OAAO,CAAC,QAAQ,GAAG,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,KAAK,QAAQ,CAAC,CAAC;IACvF,CAAC;IAED,KAAK,CAAC,MAAM,CAAC,IAAmE,EAAE;QAChF,IAAI,CAAC,IAAI,CAAC,SAAS;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACxE,MAAM,KAAK,GAAU;YACnB,cAAc,EAAE,GAAG;YACnB,QAAQ,EAAE,IAAI,CAAC,QAAQ;YACvB,SAAS,EAAE,IAAI,CAAC,OAAO;YACvB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,GAAG,EAAE,IAAI,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC1C,KAAK,EAAE,EAAE,IAAI,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE;YAChC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACtD,SAAS,EAAE,IAAI,CAAC,SAAS;YACzB,GAAG,CAAC,IAAI,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzD,OAAO,EAAE,IAAI,CAAC,OAAO,IAAI,EAAE,QAAQ,EAAE,EAAE,EAAE,QAAQ,EAAE,EAAE,EAAE;YACvD,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SACpD,CAAC;QACF,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC;QACtB,MAAM,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACzB,OAAO,KAAK,CAAC;IACf,CAAC;IAED,0DAA0D;IAC1D,QAAQ;QACN,IAAI,CAAC,IAAI,CAAC,QAAQ;YAAE,MAAM,IAAI,KAAK,CAAC,iCAAiC,CAAC,CAAC;QACvE,OAAO,QAAQ,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC;IACjC,CAAC;IAED,qCAAqC;IAE7B,QAAQ,CAAC,IAAgB;QAC/B,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,CAAC;YAChC,IAAI,CAAC,CAAC,IAAI,KAAK,SAAS;gBAAE,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;IAEO,WAAW,CAAC,CAAW,EAAE,IAAY,EAAE,MAAuB;QACpE,MAAM,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;QACrD,MAAM,CAAC,GAAc,EAAE,QAAQ,EAAE,CAAC,CAAC,QAAQ,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,KAAK,EAAE,CAAC;QACpE,IAAI,CAAC,CAAC,WAAW;YAAE,CAAC,CAAC,WAAW,GAAG,CAAC,CAAC,WAAW,CAAC;QACjD,MAAM,MAAM,GAAG,CAAC,CAAC,WAAW,IAAI,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,WAAW,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QAC5F,IAAI,MAAM,KAAK,SAAS;YAAE,CAAC,CAAC,WAAW,GAAG,MAAM,CAAC;QACjD,MAAM,EAAE,GAAG,YAAY,CAAC,CAAC,CAAC,QAAQ,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC,YAAY,CAAC,CAAC;QACjE,IAAI,EAAE;YAAE,CAAC,CAAC,QAAQ,GAAG,EAAE,CAAC;QACxB,MAAM,CAAC,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,KAAK,EAAE,CAAC,EAAE,IAAI,CAAC,CAAC;QACjD,IAAI,CAAC,KAAK,SAAS;YAAE,CAAC,CAAC,aAAa,GAAG,CAAC,CAAC;QACzC,IAAI,IAAI,CAAC,CAAC,CAAC,WAAW,IAAI,IAAI,KAAK,SAAS;YAAE,CAAC,CAAC,IAAI,GAAG,IAAI,CAAC;QAC5D,OAAO,CAAC,CAAC;IACX,CAAC;CACF;AAED,SAAS,YAAY,CACnB,EAAwB,EACxB,MAAkC,EAClC,IAAqC;IAErC,IAAI,CAAC,EAAE;QAAE,OAAO,SAAS,CAAC;IAC1B,IAAI,IAAI,KAAK,KAAK;QAAE,OAAO,EAAE,CAAC;IAC9B,MAAM,MAAM,GAAG,IAAI,GAAG,CAAS,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC;IAChF,IAAI,MAAM,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,SAAS,CAAC;IACxC,MAAM,GAAG,GAAuC,EAAE,CAAC;IACnD,KAAK,MAAM,CAAC,IAAI,MAAM;QAAE,IAAI,CAAC,IAAI,EAAE;YAAE,GAAG,CAAC,CAAC,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC,CAAC;IACpD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,YAAY,CAAC,KAAwB,EAAE,CAAW,EAAE,IAAwB;IAC1F,IAAI,CAAC,KAAK;QAAE,OAAO,SAAS,CAAC;IAC7B,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;QAAE,OAAO,KAAK,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC;IACxE,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU;QAAE,OAAO,CAAC,CAAC,WAAW,KAAK,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC,YAAY,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC;IAC3H,IAAI,IAAI,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IACzC,MAAM,CAAC,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;IAC7B,OAAO,KAAK,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC;AAC/D,CAAC"}
|
package/dist/sinks.d.ts
ADDED
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { TraceSink } from "./profiler.js";
|
|
2
|
+
/** Writes each trace to <dir>/<trace_id>.json. The zero-infrastructure default. */
|
|
3
|
+
export declare function fileSink(dir: string): TraceSink;
|
|
4
|
+
/** POSTs each trace to the ingestion API (Week 1, later). Fails soft: never breaks the app. */
|
|
5
|
+
export declare function httpSink(o: {
|
|
6
|
+
endpoint: string;
|
|
7
|
+
apiKey: string;
|
|
8
|
+
fetchImpl?: typeof fetch;
|
|
9
|
+
onError?: (e: unknown) => void;
|
|
10
|
+
}): TraceSink;
|
|
11
|
+
export declare function multiSink(...sinks: TraceSink[]): TraceSink;
|
|
12
|
+
//# sourceMappingURL=sinks.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sinks.d.ts","sourceRoot":"","sources":["../src/sinks.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,eAAe,CAAC;AAE/C,mFAAmF;AACnF,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,SAAS,CAK/C;AAED,+FAA+F;AAC/F,wBAAgB,QAAQ,CAAC,CAAC,EAAE;IAAE,QAAQ,EAAE,MAAM,CAAC;IAAC,MAAM,EAAE,MAAM,CAAC;IAAC,SAAS,CAAC,EAAE,OAAO,KAAK,CAAC;IAAC,OAAO,CAAC,EAAE,CAAC,CAAC,EAAE,OAAO,KAAK,IAAI,CAAA;CAAE,GAAG,SAAS,CAcrI;AAED,wBAAgB,SAAS,CAAC,GAAG,KAAK,EAAE,SAAS,EAAE,GAAG,SAAS,CAI1D"}
|
package/dist/sinks.js
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { mkdirSync, writeFileSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
/** Writes each trace to <dir>/<trace_id>.json. The zero-infrastructure default. */
|
|
4
|
+
export function fileSink(dir) {
|
|
5
|
+
mkdirSync(dir, { recursive: true });
|
|
6
|
+
return (trace) => {
|
|
7
|
+
writeFileSync(join(dir, `${safe(trace.trace_id)}.json`), JSON.stringify(trace, null, 2) + "\n");
|
|
8
|
+
};
|
|
9
|
+
}
|
|
10
|
+
/** POSTs each trace to the ingestion API (Week 1, later). Fails soft: never breaks the app. */
|
|
11
|
+
export function httpSink(o) {
|
|
12
|
+
const f = o.fetchImpl ?? fetch;
|
|
13
|
+
return async (trace) => {
|
|
14
|
+
try {
|
|
15
|
+
const res = await f(`${o.endpoint.replace(/\/$/, "")}/v0/traces`, {
|
|
16
|
+
method: "POST",
|
|
17
|
+
headers: { "content-type": "application/json", authorization: `Bearer ${o.apiKey}` },
|
|
18
|
+
body: JSON.stringify(trace),
|
|
19
|
+
});
|
|
20
|
+
if (!res.ok)
|
|
21
|
+
throw new Error(`chunktrace ingest failed: ${res.status}`);
|
|
22
|
+
}
|
|
23
|
+
catch (e) {
|
|
24
|
+
(o.onError ?? ((err) => console.warn(String(err))))(e);
|
|
25
|
+
}
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
export function multiSink(...sinks) {
|
|
29
|
+
return async (t) => {
|
|
30
|
+
for (const s of sinks)
|
|
31
|
+
await s(t);
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
const safe = (s) => s.replace(/[^a-zA-Z0-9._-]/g, "_");
|
|
35
|
+
//# sourceMappingURL=sinks.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"sinks.js","sourceRoot":"","sources":["../src/sinks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,SAAS,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AACnD,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAIjC,mFAAmF;AACnF,MAAM,UAAU,QAAQ,CAAC,GAAW;IAClC,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;IACpC,OAAO,CAAC,KAAY,EAAE,EAAE;QACtB,aAAa,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,OAAO,CAAC,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC;IAClG,CAAC,CAAC;AACJ,CAAC;AAED,+FAA+F;AAC/F,MAAM,UAAU,QAAQ,CAAC,CAAiG;IACxH,MAAM,CAAC,GAAG,CAAC,CAAC,SAAS,IAAI,KAAK,CAAC;IAC/B,OAAO,KAAK,EAAE,KAAY,EAAE,EAAE;QAC5B,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,MAAM,CAAC,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,YAAY,EAAE;gBAChE,MAAM,EAAE,MAAM;gBACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE,aAAa,EAAE,UAAU,CAAC,CAAC,MAAM,EAAE,EAAE;gBACpF,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC;aAC5B,CAAC,CAAC;YACH,IAAI,CAAC,GAAG,CAAC,EAAE;gBAAE,MAAM,IAAI,KAAK,CAAC,6BAA6B,GAAG,CAAC,MAAM,EAAE,CAAC,CAAC;QAC1E,CAAC;QAAC,OAAO,CAAC,EAAE,CAAC;YACX,CAAC,CAAC,CAAC,OAAO,IAAI,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;QACzD,CAAC;IACH,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,SAAS,CAAC,GAAG,KAAkB;IAC7C,OAAO,KAAK,EAAE,CAAC,EAAE,EAAE;QACjB,KAAK,MAAM,CAAC,IAAI,KAAK;YAAE,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;IACpC,CAAC,CAAC;AACJ,CAAC;AAED,MAAM,IAAI,GAAG,CAAC,CAAS,EAAE,EAAE,CAAC,CAAC,CAAC,OAAO,CAAC,kBAAkB,EAAE,GAAG,CAAC,CAAC"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,31 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@chunktrace/sdk",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.9.0",
|
|
4
|
+
"description": "Instrumentation SDK: profiler, vector-store adapters and trace sinks.",
|
|
5
|
+
"author": "Hathor Software, LLC",
|
|
6
|
+
"repository": {
|
|
7
|
+
"type": "git",
|
|
8
|
+
"url": "git+https://github.com/chunktrace/chunktrace.git",
|
|
9
|
+
"directory": "packages/sdk"
|
|
10
|
+
},
|
|
11
|
+
"homepage": "https://chunktrace.com/docs/",
|
|
12
|
+
"type": "module",
|
|
13
|
+
"main": "./dist/index.js",
|
|
14
|
+
"types": "./dist/index.d.ts",
|
|
15
|
+
"exports": {
|
|
16
|
+
".": {
|
|
17
|
+
"types": "./dist/index.d.ts",
|
|
18
|
+
"default": "./dist/index.js"
|
|
19
|
+
}
|
|
20
|
+
},
|
|
21
|
+
"files": [
|
|
22
|
+
"dist"
|
|
23
|
+
],
|
|
24
|
+
"dependencies": {
|
|
25
|
+
"@chunktrace/core": "0.9.0"
|
|
26
|
+
},
|
|
27
|
+
"publishConfig": {
|
|
28
|
+
"access": "public"
|
|
29
|
+
},
|
|
30
|
+
"license": "MIT"
|
|
31
|
+
}
|