@grackle-ai/knowledge 0.62.1 → 0.63.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +4 -24
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +6 -16
- package/dist/index.js.map +1 -1
- package/dist/reference-sync.d.ts +1 -2
- package/dist/reference-sync.d.ts.map +1 -1
- package/dist/reference-sync.js +1 -5
- package/dist/reference-sync.js.map +1 -1
- package/package.json +2 -5
- package/dist/chunker.d.ts +0 -20
- package/dist/chunker.d.ts.map +0 -1
- package/dist/chunker.js +0 -7
- package/dist/chunker.js.map +0 -1
- package/dist/client.d.ts +0 -68
- package/dist/client.d.ts.map +0 -1
- package/dist/client.js +0 -150
- package/dist/client.js.map +0 -1
- package/dist/constants.d.ts +0 -33
- package/dist/constants.d.ts.map +0 -1
- package/dist/constants.js +0 -33
- package/dist/constants.js.map +0 -1
- package/dist/edge-store.d.ts +0 -32
- package/dist/edge-store.d.ts.map +0 -1
- package/dist/edge-store.js +0 -145
- package/dist/edge-store.js.map +0 -1
- package/dist/embedder.d.ts +0 -35
- package/dist/embedder.d.ts.map +0 -1
- package/dist/embedder.js +0 -7
- package/dist/embedder.js.map +0 -1
- package/dist/expand.d.ts +0 -42
- package/dist/expand.d.ts.map +0 -1
- package/dist/expand.js +0 -150
- package/dist/expand.js.map +0 -1
- package/dist/ingest.d.ts +0 -23
- package/dist/ingest.d.ts.map +0 -1
- package/dist/ingest.js +0 -27
- package/dist/ingest.js.map +0 -1
- package/dist/local-embedder.d.ts +0 -25
- package/dist/local-embedder.d.ts.map +0 -1
- package/dist/local-embedder.js +0 -88
- package/dist/local-embedder.js.map +0 -1
- package/dist/logger.d.ts +0 -9
- package/dist/logger.d.ts.map +0 -1
- package/dist/logger.js +0 -15
- package/dist/logger.js.map +0 -1
- package/dist/node-store.d.ts +0 -116
- package/dist/node-store.d.ts.map +0 -1
- package/dist/node-store.js +0 -268
- package/dist/node-store.js.map +0 -1
- package/dist/pass-through-chunker.d.ts +0 -16
- package/dist/pass-through-chunker.d.ts.map +0 -1
- package/dist/pass-through-chunker.js +0 -21
- package/dist/pass-through-chunker.js.map +0 -1
- package/dist/schema.d.ts +0 -23
- package/dist/schema.d.ts.map +0 -1
- package/dist/schema.js +0 -73
- package/dist/schema.js.map +0 -1
- package/dist/search.d.ts +0 -43
- package/dist/search.d.ts.map +0 -1
- package/dist/search.js +0 -123
- package/dist/search.js.map +0 -1
- package/dist/transcript-chunker.d.ts +0 -29
- package/dist/transcript-chunker.d.ts.map +0 -1
- package/dist/transcript-chunker.js +0 -142
- package/dist/transcript-chunker.js.map +0 -1
- package/dist/types.d.ts +0 -102
- package/dist/types.d.ts.map +0 -1
- package/dist/types.js +0 -61
- package/dist/types.js.map +0 -1
package/dist/edge-store.js
DELETED
|
@@ -1,145 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Edge CRUD operations for the knowledge graph.
|
|
3
|
-
*
|
|
4
|
-
* Provides create and remove operations for typed relationships between
|
|
5
|
-
* {@link KnowledgeNode} instances, backed by Neo4j.
|
|
6
|
-
*
|
|
7
|
-
* @module
|
|
8
|
-
*/
|
|
9
|
-
import { getSession } from "./client.js";
|
|
10
|
-
import { logger } from "./logger.js";
|
|
11
|
-
import { NODE_LABEL } from "./constants.js";
|
|
12
|
-
import { EDGE_TYPE } from "./types.js";
|
|
13
|
-
// ---------------------------------------------------------------------------
|
|
14
|
-
// Validation
|
|
15
|
-
// ---------------------------------------------------------------------------
|
|
16
|
-
/** Known edge type values for runtime validation. */
|
|
17
|
-
const VALID_EDGE_TYPES = new Set(Object.values(EDGE_TYPE));
|
|
18
|
-
/**
|
|
19
|
-
* Assert that a string is a valid {@link EdgeType}.
|
|
20
|
-
*
|
|
21
|
-
* @throws If the value is not a known edge type.
|
|
22
|
-
*/
|
|
23
|
-
function assertValidEdgeType(type) {
|
|
24
|
-
if (!VALID_EDGE_TYPES.has(type)) {
|
|
25
|
-
throw new Error(`Invalid edge type: "${type}". Must be one of: ${[...VALID_EDGE_TYPES].join(", ")}`);
|
|
26
|
-
}
|
|
27
|
-
}
|
|
28
|
-
// ---------------------------------------------------------------------------
|
|
29
|
-
// Cypher builders
|
|
30
|
-
// ---------------------------------------------------------------------------
|
|
31
|
-
/**
|
|
32
|
-
* Build the Cypher query for creating an edge of a given type.
|
|
33
|
-
*
|
|
34
|
-
* Relationship types cannot be parameterized in Cypher, so the type is
|
|
35
|
-
* interpolated directly. This is safe because {@link assertValidEdgeType}
|
|
36
|
-
* ensures the value comes from the closed {@link EdgeType} union.
|
|
37
|
-
*/
|
|
38
|
-
function buildCreateEdgeCypher(edgeType) {
|
|
39
|
-
return [
|
|
40
|
-
`MATCH (a:${NODE_LABEL} {id: $fromId}), (b:${NODE_LABEL} {id: $toId})`,
|
|
41
|
-
`CREATE (a)-[r:${edgeType} {metadata: $metadata, createdAt: $createdAt}]->(b)`,
|
|
42
|
-
`RETURN a.id AS fromId, b.id AS toId, type(r) AS type, r.metadata AS metadata, r.createdAt AS createdAt`,
|
|
43
|
-
].join("\n");
|
|
44
|
-
}
|
|
45
|
-
/**
|
|
46
|
-
* Build the Cypher query for removing an edge of a given type.
|
|
47
|
-
*/
|
|
48
|
-
function buildRemoveEdgeCypher(edgeType) {
|
|
49
|
-
return [
|
|
50
|
-
`MATCH (a:${NODE_LABEL} {id: $fromId})-[r:${edgeType}]->(b:${NODE_LABEL} {id: $toId})`,
|
|
51
|
-
`DELETE r`,
|
|
52
|
-
`RETURN count(r) AS deleted`,
|
|
53
|
-
].join("\n");
|
|
54
|
-
}
|
|
55
|
-
// ---------------------------------------------------------------------------
|
|
56
|
-
// Public API
|
|
57
|
-
// ---------------------------------------------------------------------------
|
|
58
|
-
/**
|
|
59
|
-
* Create a typed edge between two nodes.
|
|
60
|
-
*
|
|
61
|
-
* @param fromId - Source node ID.
|
|
62
|
-
* @param toId - Target node ID.
|
|
63
|
-
* @param type - Relationship type (must be a valid {@link EdgeType}).
|
|
64
|
-
* @param metadata - Optional metadata to attach to the edge.
|
|
65
|
-
* @returns The created edge.
|
|
66
|
-
* @throws If either node does not exist.
|
|
67
|
-
* @throws If the edge type is invalid.
|
|
68
|
-
*/
|
|
69
|
-
export async function createEdge(fromId, toId, type, metadata) {
|
|
70
|
-
assertValidEdgeType(type);
|
|
71
|
-
const createdAt = new Date().toISOString();
|
|
72
|
-
const metadataStr = metadata !== undefined ? JSON.stringify(metadata) : null;
|
|
73
|
-
const session = getSession();
|
|
74
|
-
try {
|
|
75
|
-
const result = await session.run(buildCreateEdgeCypher(type), {
|
|
76
|
-
fromId,
|
|
77
|
-
toId,
|
|
78
|
-
metadata: metadataStr,
|
|
79
|
-
createdAt,
|
|
80
|
-
});
|
|
81
|
-
if (result.records.length === 0) {
|
|
82
|
-
throw new Error(`Cannot create edge: one or both nodes not found (fromId=${fromId}, toId=${toId})`);
|
|
83
|
-
}
|
|
84
|
-
const record = result.records[0];
|
|
85
|
-
let parsedMetadata;
|
|
86
|
-
const rawMetadata = record.get("metadata");
|
|
87
|
-
if (rawMetadata !== null) {
|
|
88
|
-
try {
|
|
89
|
-
parsedMetadata = JSON.parse(rawMetadata);
|
|
90
|
-
}
|
|
91
|
-
catch {
|
|
92
|
-
parsedMetadata = undefined;
|
|
93
|
-
}
|
|
94
|
-
}
|
|
95
|
-
logger.debug({ fromId, toId, type }, "Created edge");
|
|
96
|
-
return {
|
|
97
|
-
fromId: record.get("fromId"),
|
|
98
|
-
toId: record.get("toId"),
|
|
99
|
-
type: record.get("type"),
|
|
100
|
-
metadata: parsedMetadata,
|
|
101
|
-
createdAt: record.get("createdAt"),
|
|
102
|
-
};
|
|
103
|
-
}
|
|
104
|
-
finally {
|
|
105
|
-
try {
|
|
106
|
-
await session.close();
|
|
107
|
-
}
|
|
108
|
-
catch (closeError) {
|
|
109
|
-
logger.warn({ err: closeError }, "Failed to close session after createEdge");
|
|
110
|
-
}
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
/**
|
|
114
|
-
* Remove an edge between two nodes.
|
|
115
|
-
*
|
|
116
|
-
* @param fromId - Source node ID.
|
|
117
|
-
* @param toId - Target node ID.
|
|
118
|
-
* @param type - Relationship type to remove.
|
|
119
|
-
* @returns `true` if an edge was removed, `false` if no matching edge existed.
|
|
120
|
-
* @throws If the edge type is invalid.
|
|
121
|
-
*/
|
|
122
|
-
export async function removeEdge(fromId, toId, type) {
|
|
123
|
-
assertValidEdgeType(type);
|
|
124
|
-
const session = getSession();
|
|
125
|
-
try {
|
|
126
|
-
const result = await session.run(buildRemoveEdgeCypher(type), {
|
|
127
|
-
fromId,
|
|
128
|
-
toId,
|
|
129
|
-
});
|
|
130
|
-
const deleted = result.records[0]?.get("deleted");
|
|
131
|
-
if (deleted > 0) {
|
|
132
|
-
logger.debug({ fromId, toId, type }, "Removed edge");
|
|
133
|
-
}
|
|
134
|
-
return deleted > 0;
|
|
135
|
-
}
|
|
136
|
-
finally {
|
|
137
|
-
try {
|
|
138
|
-
await session.close();
|
|
139
|
-
}
|
|
140
|
-
catch (closeError) {
|
|
141
|
-
logger.warn({ err: closeError }, "Failed to close session after removeEdge");
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
|
-
}
|
|
145
|
-
//# sourceMappingURL=edge-store.js.map
|
package/dist/edge-store.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"edge-store.js","sourceRoot":"","sources":["../src/edge-store.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAEH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,SAAS,EAAqC,MAAM,YAAY,CAAC;AAE1E,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E,qDAAqD;AACrD,MAAM,gBAAgB,GAAwB,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;AAEhF;;;;GAIG;AACH,SAAS,mBAAmB,CAAC,IAAY;IACvC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,CAAC;QAChC,MAAM,IAAI,KAAK,CACb,uBAAuB,IAAI,sBAAsB,CAAC,GAAG,gBAAgB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACpF,CAAC;IACJ,CAAC;AACH,CAAC;AAED,8EAA8E;AAC9E,kBAAkB;AAClB,8EAA8E;AAE9E;;;;;;GAMG;AACH,SAAS,qBAAqB,CAAC,QAAkB;IAC/C,OAAO;QACL,YAAY,UAAU,uBAAuB,UAAU,eAAe;QACtE,iBAAiB,QAAQ,qDAAqD;QAC9E,wGAAwG;KACzG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED;;GAEG;AACH,SAAS,qBAAqB,CAAC,QAAkB;IAC/C,OAAO;QACL,YAAY,UAAU,sBAAsB,QAAQ,SAAS,UAAU,eAAe;QACtF,UAAU;QACV,4BAA4B;KAC7B,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AACf,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E;;;;;;;;;;GAUG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAAc,EACd,IAAY,EACZ,IAAc,EACd,QAAkC;IAElC,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAE1B,MAAM,SAAS,GAAG,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC;IAC3C,MAAM,WAAW,GAAkB,QAAQ,KAAK,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAE5F,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,CAAC,EAAE;YAC5D,MAAM;YACN,IAAI;YACJ,QAAQ,EAAE,WAAW;YACrB,SAAS;SACV,CAAC,CAAC;QAEH,IAAI,MAAM,CAAC,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAChC,MAAM,IAAI,KAAK,CACb,2DAA2D,MAAM,UAAU,IAAI,GAAG,CACnF,CAAC;QACJ,CAAC;QAED,MAAM,MAAM,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;QAEjC,IAAI,cAAmD,CAAC;QACxD,MAAM,WAAW,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAAkB,CAAC;QAC5D,IAAI,WAAW,KAAK,IAAI,EAAE,CAAC;YACzB,IAAI,CAAC;gBACH,cAAc,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAA4B,CAAC;YACtE,CAAC;YAAC,MAAM,CAAC;gBACP,cAAc,GAAG,SAAS,CAAC;YAC7B,CAAC;QACH,CAAC;QAED,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,cAAc,CAAC,CAAC;QAErD,OAAO;YACL,MAAM,EAAE,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAW;YACtC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAW;YAClC,IAAI,EAAE,MAAM,CAAC,GAAG,CAAC,MAAM,CAAa;YACpC,QAAQ,EAAE,cAAc;YACxB,SAAS,EAAE,MAAM,CAAC,GAAG,CAAC,WAAW,CAAW;SAC7C,CAAC;IACJ,CAAC;YAAS,CAAC;QACT,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;QAAC,OAAO,UAAU,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,0CAA0C,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAAc,EACd,IAAY,EACZ,IAAc;IAEd,mBAAmB,CAAC,IAAI,CAAC,CAAC;IAE1B,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,IAAI,CAAC,EAAE;YAC5D,MAAM;YACN,IAAI;SACL,CAAC,CAAC;QAEH,MAAM,OAAO,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,GAAG,CAAC,SAAS,CAAW,CAAC;QAC5D,IAAI,OAAO,GAAG,CAAC,EAAE,CAAC;YAChB,MAAM,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,IAAI,EAAE,EAAE,cAAc,CAAC,CAAC;QACvD,CAAC;QACD,OAAO,OAAO,GAAG,CAAC,CAAC;IACrB,CAAC;YAAS,CAAC;QACT,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;QAAC,OAAO,UAAU,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,0CAA0C,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;AACH,CAAC"}
|
package/dist/embedder.d.ts
DELETED
|
@@ -1,35 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Pluggable embedder interface for converting text into vector embeddings.
|
|
3
|
-
*
|
|
4
|
-
* @module
|
|
5
|
-
*/
|
|
6
|
-
/** A single embedding result pairing input text with its vector. */
|
|
7
|
-
export interface EmbeddingResult {
|
|
8
|
-
/** The input text that was embedded. */
|
|
9
|
-
text: string;
|
|
10
|
-
/** The embedding vector (array of floats). */
|
|
11
|
-
vector: number[];
|
|
12
|
-
}
|
|
13
|
-
/** Configuration for creating an {@link Embedder}. */
|
|
14
|
-
export interface EmbedderOptions {
|
|
15
|
-
/** HuggingFace model ID (e.g., `"Xenova/all-MiniLM-L6-v2"`). */
|
|
16
|
-
modelId?: string;
|
|
17
|
-
/** Expected embedding dimensions. Used for validation when set. */
|
|
18
|
-
dimensions?: number;
|
|
19
|
-
}
|
|
20
|
-
/** Converts text into embedding vectors for semantic search and similarity. */
|
|
21
|
-
export interface Embedder {
|
|
22
|
-
/** Embed a single text string. */
|
|
23
|
-
embed(text: string): Promise<EmbeddingResult>;
|
|
24
|
-
/** Embed multiple texts in batch. */
|
|
25
|
-
embedBatch(texts: string[]): Promise<EmbeddingResult[]>;
|
|
26
|
-
/**
|
|
27
|
-
* The dimensionality of vectors produced by this embedder.
|
|
28
|
-
*
|
|
29
|
-
* When using a custom model without specifying `dimensions` in options,
|
|
30
|
-
* this value is `0` until the first embedding is computed. A value of `0`
|
|
31
|
-
* means the true dimensions are not yet known.
|
|
32
|
-
*/
|
|
33
|
-
readonly dimensions: number;
|
|
34
|
-
}
|
|
35
|
-
//# sourceMappingURL=embedder.d.ts.map
|
package/dist/embedder.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,oEAAoE;AACpE,MAAM,WAAW,eAAe;IAC9B,wCAAwC;IACxC,IAAI,EAAE,MAAM,CAAC;IACb,8CAA8C;IAC9C,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED,sDAAsD;AACtD,MAAM,WAAW,eAAe;IAC9B,gEAAgE;IAChE,OAAO,CAAC,EAAE,MAAM,CAAC;IACjB,mEAAmE;IACnE,UAAU,CAAC,EAAE,MAAM,CAAC;CACrB;AAED,+EAA+E;AAC/E,MAAM,WAAW,QAAQ;IACvB,kCAAkC;IAClC,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,eAAe,CAAC,CAAC;IAC9C,qCAAqC;IACrC,UAAU,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,eAAe,EAAE,CAAC,CAAC;IACxD;;;;;;OAMG;IACH,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;CAC7B"}
|
package/dist/embedder.js
DELETED
package/dist/embedder.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"embedder.js","sourceRoot":"","sources":["../src/embedder.ts"],"names":[],"mappings":"AAAA;;;;GAIG"}
|
package/dist/expand.d.ts
DELETED
|
@@ -1,42 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Graph expansion — traverse the knowledge graph from a starting node
|
|
3
|
-
* to discover connected nodes within N hops.
|
|
4
|
-
*
|
|
5
|
-
* @module
|
|
6
|
-
*/
|
|
7
|
-
import type { KnowledgeNode, KnowledgeEdge, EdgeType } from "./types.js";
|
|
8
|
-
import type { SearchResult } from "./search.js";
|
|
9
|
-
/** Options for graph expansion. */
|
|
10
|
-
export interface ExpandOptions {
|
|
11
|
-
/** Max traversal depth (default 1). */
|
|
12
|
-
depth?: number;
|
|
13
|
-
/** Only follow edges of these types. If empty/undefined, follow all. */
|
|
14
|
-
edgeTypes?: EdgeType[];
|
|
15
|
-
}
|
|
16
|
-
/** Result of a graph expansion: the subgraph reachable from the starting node(s). */
|
|
17
|
-
export interface ExpansionResult {
|
|
18
|
-
/** All neighbor nodes found (deduplicated), excluding the start node(s). */
|
|
19
|
-
nodes: KnowledgeNode[];
|
|
20
|
-
/** All edges traversed. */
|
|
21
|
-
edges: KnowledgeEdge[];
|
|
22
|
-
}
|
|
23
|
-
/**
|
|
24
|
-
* Expand a single node: return its neighbors within N hops.
|
|
25
|
-
*
|
|
26
|
-
* @param nodeId - The starting node ID.
|
|
27
|
-
* @param options - Traversal depth and optional edge type filter.
|
|
28
|
-
* @returns Neighbor nodes and traversed edges (deduplicated).
|
|
29
|
-
*/
|
|
30
|
-
export declare function expandNode(nodeId: string, options?: ExpandOptions): Promise<ExpansionResult>;
|
|
31
|
-
/**
|
|
32
|
-
* Expand search results: return neighbors of all result nodes, deduplicated.
|
|
33
|
-
*
|
|
34
|
-
* Runs {@link expandNode} for each search result and merges the subgraphs,
|
|
35
|
-
* excluding the original search result nodes from the neighbor list.
|
|
36
|
-
*
|
|
37
|
-
* @param results - Search results from {@link knowledgeSearch}.
|
|
38
|
-
* @param options - Traversal depth and optional edge type filter.
|
|
39
|
-
* @returns Combined neighbor nodes and edges from all start nodes.
|
|
40
|
-
*/
|
|
41
|
-
export declare function expandResults(results: SearchResult[], options?: ExpandOptions): Promise<ExpansionResult>;
|
|
42
|
-
//# sourceMappingURL=expand.d.ts.map
|
package/dist/expand.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"expand.d.ts","sourceRoot":"","sources":["../src/expand.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAOH,OAAO,KAAK,EAAE,aAAa,EAAE,aAAa,EAAE,QAAQ,EAAE,MAAM,YAAY,CAAC;AAEzE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAMhD,mCAAmC;AACnC,MAAM,WAAW,aAAa;IAC5B,uCAAuC;IACvC,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,wEAAwE;IACxE,SAAS,CAAC,EAAE,QAAQ,EAAE,CAAC;CACxB;AAED,qFAAqF;AACrF,MAAM,WAAW,eAAe;IAC9B,4EAA4E;IAC5E,KAAK,EAAE,aAAa,EAAE,CAAC;IACvB,2BAA2B;IAC3B,KAAK,EAAE,aAAa,EAAE,CAAC;CACxB;AAiFD;;;;;;GAMG;AACH,wBAAsB,UAAU,CAC9B,MAAM,EAAE,MAAM,EACd,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,eAAe,CAAC,CA8C1B;AAED;;;;;;;;;GASG;AACH,wBAAsB,aAAa,CACjC,OAAO,EAAE,YAAY,EAAE,EACvB,OAAO,CAAC,EAAE,aAAa,GACtB,OAAO,CAAC,eAAe,CAAC,CAiB1B"}
|
package/dist/expand.js
DELETED
|
@@ -1,150 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Graph expansion — traverse the knowledge graph from a starting node
|
|
3
|
-
* to discover connected nodes within N hops.
|
|
4
|
-
*
|
|
5
|
-
* @module
|
|
6
|
-
*/
|
|
7
|
-
import { getSession } from "./client.js";
|
|
8
|
-
import { logger } from "./logger.js";
|
|
9
|
-
import { NODE_LABEL } from "./constants.js";
|
|
10
|
-
import { recordToNode, recordToEdge } from "./node-store.js";
|
|
11
|
-
import { EDGE_TYPE } from "./types.js";
|
|
12
|
-
// ---------------------------------------------------------------------------
|
|
13
|
-
// Constants
|
|
14
|
-
// ---------------------------------------------------------------------------
|
|
15
|
-
/** Default traversal depth. */
|
|
16
|
-
const DEFAULT_DEPTH = 1;
|
|
17
|
-
/** Known edge type values for runtime validation. */
|
|
18
|
-
const VALID_EDGE_TYPES = new Set(Object.values(EDGE_TYPE));
|
|
19
|
-
// ---------------------------------------------------------------------------
|
|
20
|
-
// Cypher
|
|
21
|
-
// ---------------------------------------------------------------------------
|
|
22
|
-
/**
|
|
23
|
-
* Build the Cypher query for variable-length path traversal.
|
|
24
|
-
*
|
|
25
|
-
* When edge types are specified, they are interpolated into the relationship
|
|
26
|
-
* pattern (e.g., `[:RELATES_TO|DEPENDS_ON*1..2]`). This is safe because
|
|
27
|
-
* values are validated against the closed {@link EdgeType} union.
|
|
28
|
-
*/
|
|
29
|
-
function buildExpandCypher(depth, edgeTypes) {
|
|
30
|
-
// Neo4j requires integer literals for variable-length ranges — cannot parameterize.
|
|
31
|
-
let relPattern;
|
|
32
|
-
if (edgeTypes && edgeTypes.length > 0) {
|
|
33
|
-
for (const t of edgeTypes) {
|
|
34
|
-
if (!VALID_EDGE_TYPES.has(t)) {
|
|
35
|
-
throw new Error(`Invalid edge type: "${t}". Must be one of: ${[...VALID_EDGE_TYPES].join(", ")}`);
|
|
36
|
-
}
|
|
37
|
-
}
|
|
38
|
-
relPattern = `[:${edgeTypes.join("|")}*1..${depth}]`;
|
|
39
|
-
}
|
|
40
|
-
else {
|
|
41
|
-
relPattern = `[*1..${depth}]`;
|
|
42
|
-
}
|
|
43
|
-
return `
|
|
44
|
-
MATCH path = (start:${NODE_LABEL} {id: $startId})-${relPattern}-(neighbor:${NODE_LABEL})
|
|
45
|
-
WHERE neighbor.id <> $startId
|
|
46
|
-
UNWIND relationships(path) AS rel
|
|
47
|
-
WITH DISTINCT neighbor,
|
|
48
|
-
collect(DISTINCT {
|
|
49
|
-
fromId: startNode(rel).id,
|
|
50
|
-
toId: endNode(rel).id,
|
|
51
|
-
type: type(rel),
|
|
52
|
-
metadata: rel.metadata,
|
|
53
|
-
createdAt: rel.createdAt
|
|
54
|
-
}) AS rels
|
|
55
|
-
RETURN neighbor, rels`;
|
|
56
|
-
}
|
|
57
|
-
/**
|
|
58
|
-
* Merge two expansion results, deduplicating nodes by ID and edges by
|
|
59
|
-
* (fromId, toId, type) triple.
|
|
60
|
-
*/
|
|
61
|
-
function mergeResults(a, b) {
|
|
62
|
-
const nodeMap = new Map();
|
|
63
|
-
for (const node of [...a.nodes, ...b.nodes]) {
|
|
64
|
-
nodeMap.set(node.id, node);
|
|
65
|
-
}
|
|
66
|
-
const edgeSet = new Set();
|
|
67
|
-
const edges = [];
|
|
68
|
-
for (const edge of [...a.edges, ...b.edges]) {
|
|
69
|
-
const key = `${edge.fromId}:${edge.toId}:${edge.type}`;
|
|
70
|
-
if (!edgeSet.has(key)) {
|
|
71
|
-
edgeSet.add(key);
|
|
72
|
-
edges.push(edge);
|
|
73
|
-
}
|
|
74
|
-
}
|
|
75
|
-
return { nodes: [...nodeMap.values()], edges };
|
|
76
|
-
}
|
|
77
|
-
// ---------------------------------------------------------------------------
|
|
78
|
-
// Public API
|
|
79
|
-
// ---------------------------------------------------------------------------
|
|
80
|
-
/**
|
|
81
|
-
* Expand a single node: return its neighbors within N hops.
|
|
82
|
-
*
|
|
83
|
-
* @param nodeId - The starting node ID.
|
|
84
|
-
* @param options - Traversal depth and optional edge type filter.
|
|
85
|
-
* @returns Neighbor nodes and traversed edges (deduplicated).
|
|
86
|
-
*/
|
|
87
|
-
export async function expandNode(nodeId, options) {
|
|
88
|
-
const rawDepth = options?.depth ?? DEFAULT_DEPTH;
|
|
89
|
-
const depth = Number.isFinite(rawDepth) ? Math.max(1, Math.floor(rawDepth)) : DEFAULT_DEPTH;
|
|
90
|
-
const cypher = buildExpandCypher(depth, options?.edgeTypes);
|
|
91
|
-
const session = getSession();
|
|
92
|
-
try {
|
|
93
|
-
const result = await session.run(cypher, { startId: nodeId });
|
|
94
|
-
const nodeMap = new Map();
|
|
95
|
-
const edgeSet = new Set();
|
|
96
|
-
const edges = [];
|
|
97
|
-
for (const record of result.records) {
|
|
98
|
-
const neo4jNode = record.get("neighbor");
|
|
99
|
-
const node = recordToNode(neo4jNode.properties);
|
|
100
|
-
nodeMap.set(node.id, node);
|
|
101
|
-
const rawEdges = record.get("rels");
|
|
102
|
-
for (const raw of rawEdges) {
|
|
103
|
-
if (raw.fromId === null || raw.toId === null) {
|
|
104
|
-
continue;
|
|
105
|
-
}
|
|
106
|
-
const edge = recordToEdge(raw);
|
|
107
|
-
const key = `${edge.fromId}:${edge.toId}:${edge.type}`;
|
|
108
|
-
if (!edgeSet.has(key)) {
|
|
109
|
-
edgeSet.add(key);
|
|
110
|
-
edges.push(edge);
|
|
111
|
-
}
|
|
112
|
-
}
|
|
113
|
-
}
|
|
114
|
-
logger.debug({ startId: nodeId, depth, nodes: nodeMap.size, edges: edges.length }, "Graph expansion completed");
|
|
115
|
-
return { nodes: [...nodeMap.values()], edges };
|
|
116
|
-
}
|
|
117
|
-
finally {
|
|
118
|
-
try {
|
|
119
|
-
await session.close();
|
|
120
|
-
}
|
|
121
|
-
catch (closeError) {
|
|
122
|
-
logger.warn({ err: closeError }, "Failed to close session after expandNode");
|
|
123
|
-
}
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
/**
|
|
127
|
-
* Expand search results: return neighbors of all result nodes, deduplicated.
|
|
128
|
-
*
|
|
129
|
-
* Runs {@link expandNode} for each search result and merges the subgraphs,
|
|
130
|
-
* excluding the original search result nodes from the neighbor list.
|
|
131
|
-
*
|
|
132
|
-
* @param results - Search results from {@link knowledgeSearch}.
|
|
133
|
-
* @param options - Traversal depth and optional edge type filter.
|
|
134
|
-
* @returns Combined neighbor nodes and edges from all start nodes.
|
|
135
|
-
*/
|
|
136
|
-
export async function expandResults(results, options) {
|
|
137
|
-
if (results.length === 0) {
|
|
138
|
-
return { nodes: [], edges: [] };
|
|
139
|
-
}
|
|
140
|
-
const startIds = new Set(results.map((r) => r.node.id));
|
|
141
|
-
let merged = { nodes: [], edges: [] };
|
|
142
|
-
for (const result of results) {
|
|
143
|
-
const expansion = await expandNode(result.node.id, options);
|
|
144
|
-
merged = mergeResults(merged, expansion);
|
|
145
|
-
}
|
|
146
|
-
// Exclude the original search result nodes from the neighbor list
|
|
147
|
-
merged.nodes = merged.nodes.filter((n) => !startIds.has(n.id));
|
|
148
|
-
return merged;
|
|
149
|
-
}
|
|
150
|
-
//# sourceMappingURL=expand.js.map
|
package/dist/expand.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"expand.js","sourceRoot":"","sources":["../src/expand.ts"],"names":[],"mappings":"AAAA;;;;;GAKG;AAGH,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AACrC,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAC5C,OAAO,EAAE,YAAY,EAAE,YAAY,EAAE,MAAM,iBAAiB,CAAC;AAE7D,OAAO,EAAE,SAAS,EAAE,MAAM,YAAY,CAAC;AAuBvC,8EAA8E;AAC9E,YAAY;AACZ,8EAA8E;AAE9E,+BAA+B;AAC/B,MAAM,aAAa,GAAW,CAAC,CAAC;AAEhC,qDAAqD;AACrD,MAAM,gBAAgB,GAAwB,IAAI,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,CAAC;AAEhF,8EAA8E;AAC9E,SAAS;AACT,8EAA8E;AAE9E;;;;;;GAMG;AACH,SAAS,iBAAiB,CAAC,KAAa,EAAE,SAAsB;IAC9D,oFAAoF;IACpF,IAAI,UAAkB,CAAC;IACvB,IAAI,SAAS,IAAI,SAAS,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QACtC,KAAK,MAAM,CAAC,IAAI,SAAS,EAAE,CAAC;YAC1B,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;gBAC7B,MAAM,IAAI,KAAK,CACb,uBAAuB,CAAC,sBAAsB,CAAC,GAAG,gBAAgB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACjF,CAAC;YACJ,CAAC;QACH,CAAC;QACD,UAAU,GAAG,KAAK,SAAS,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,KAAK,GAAG,CAAC;IACvD,CAAC;SAAM,CAAC;QACN,UAAU,GAAG,QAAQ,KAAK,GAAG,CAAC;IAChC,CAAC;IAED,OAAO;0BACiB,UAAU,oBAAoB,UAAU,cAAc,UAAU;;;;;;;;;;;0BAWhE,CAAC;AAC3B,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,CAAkB,EAAE,CAAkB;IAC1D,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;IACjD,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5C,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;IAC7B,CAAC;IAED,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;IAClC,MAAM,KAAK,GAAoB,EAAE,CAAC;IAClC,KAAK,MAAM,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,KAAK,CAAC,EAAE,CAAC;QAC5C,MAAM,GAAG,GAAW,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;QAC/D,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACtB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnB,CAAC;IACH,CAAC;IAED,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;AACjD,CAAC;AAED,8EAA8E;AAC9E,aAAa;AACb,8EAA8E;AAE9E;;;;;;GAMG;AACH,MAAM,CAAC,KAAK,UAAU,UAAU,CAC9B,MAAc,EACd,OAAuB;IAEvB,MAAM,QAAQ,GAAW,OAAO,EAAE,KAAK,IAAI,aAAa,CAAC;IACzD,MAAM,KAAK,GAAW,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,CAAC,aAAa,CAAC;IACpG,MAAM,MAAM,GAAW,iBAAiB,CAAC,KAAK,EAAE,OAAO,EAAE,SAAS,CAAC,CAAC;IAEpE,MAAM,OAAO,GAAG,UAAU,EAAE,CAAC;IAC7B,IAAI,CAAC;QACH,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC;QAE9D,MAAM,OAAO,GAAG,IAAI,GAAG,EAAyB,CAAC;QACjD,MAAM,OAAO,GAAG,IAAI,GAAG,EAAU,CAAC;QAClC,MAAM,KAAK,GAAoB,EAAE,CAAC;QAElC,KAAK,MAAM,MAAM,IAAI,MAAM,CAAC,OAAwB,EAAE,CAAC;YACrD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,CAAC,UAAU,CAA4C,CAAC;YACpF,MAAM,IAAI,GAAkB,YAAY,CAAC,SAAS,CAAC,UAAU,CAAC,CAAC;YAC/D,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YAE3B,MAAM,QAAQ,GACZ,MAAM,CAAC,GAAG,CAAC,MAAM,CAA8B,CAAC;YAClD,KAAK,MAAM,GAAG,IAAI,QAAQ,EAAE,CAAC;gBAC3B,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,IAAI,EAAE,CAAC;oBAC7C,SAAS;gBACX,CAAC;gBACD,MAAM,IAAI,GAAkB,YAAY,CAAC,GAAG,CAAC,CAAC;gBAC9C,MAAM,GAAG,GAAW,GAAG,IAAI,CAAC,MAAM,IAAI,IAAI,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI,EAAE,CAAC;gBAC/D,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;oBACtB,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;oBACjB,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;gBACnB,CAAC;YACH,CAAC;QACH,CAAC;QAED,MAAM,CAAC,KAAK,CACV,EAAE,OAAO,EAAE,MAAM,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,IAAI,EAAE,KAAK,EAAE,KAAK,CAAC,MAAM,EAAE,EACpE,2BAA2B,CAC5B,CAAC;QAEF,OAAO,EAAE,KAAK,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,KAAK,EAAE,CAAC;IACjD,CAAC;YAAS,CAAC;QACT,IAAI,CAAC;YACH,MAAM,OAAO,CAAC,KAAK,EAAE,CAAC;QACxB,CAAC;QAAC,OAAO,UAAU,EAAE,CAAC;YACpB,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,EAAE,0CAA0C,CAAC,CAAC;QAC/E,CAAC;IACH,CAAC;AACH,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,CAAC,KAAK,UAAU,aAAa,CACjC,OAAuB,EACvB,OAAuB;IAEvB,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IAClC,CAAC;IAED,MAAM,QAAQ,GAAgB,IAAI,GAAG,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC;IACrE,IAAI,MAAM,GAAoB,EAAE,KAAK,EAAE,EAAE,EAAE,KAAK,EAAE,EAAE,EAAE,CAAC;IAEvD,KAAK,MAAM,MAAM,IAAI,OAAO,EAAE,CAAC;QAC7B,MAAM,SAAS,GAAoB,MAAM,UAAU,CAAC,MAAM,CAAC,IAAI,CAAC,EAAE,EAAE,OAAO,CAAC,CAAC;QAC7E,MAAM,GAAG,YAAY,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;IAC3C,CAAC;IAED,kEAAkE;IAClE,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC;IAE/D,OAAO,MAAM,CAAC;AAChB,CAAC"}
|
package/dist/ingest.d.ts
DELETED
|
@@ -1,23 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Ingestion pipeline that chunks content and embeds each chunk.
|
|
3
|
-
*
|
|
4
|
-
* @module
|
|
5
|
-
*/
|
|
6
|
-
import type { Chunk, Chunker } from "./chunker.js";
|
|
7
|
-
import type { Embedder } from "./embedder.js";
|
|
8
|
-
/** A chunk with its embedding vector attached. */
|
|
9
|
-
export interface EmbeddedChunk extends Chunk {
|
|
10
|
-
/** The embedding vector for this chunk's text. */
|
|
11
|
-
vector: number[];
|
|
12
|
-
}
|
|
13
|
-
/**
|
|
14
|
-
* Chunk content and embed each chunk in a single pipeline.
|
|
15
|
-
*
|
|
16
|
-
* @param content - The raw text content to ingest.
|
|
17
|
-
* @param chunker - Splits the content into chunks.
|
|
18
|
-
* @param embedder - Produces embedding vectors for each chunk.
|
|
19
|
-
* @param metadata - Optional metadata passed through to the chunker.
|
|
20
|
-
* @returns An array of chunks with embedding vectors attached.
|
|
21
|
-
*/
|
|
22
|
-
export declare function ingest(content: string, chunker: Chunker, embedder: Embedder, metadata?: Record<string, unknown>): Promise<EmbeddedChunk[]>;
|
|
23
|
-
//# sourceMappingURL=ingest.d.ts.map
|
package/dist/ingest.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ingest.d.ts","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AACnD,OAAO,KAAK,EAAE,QAAQ,EAAE,MAAM,eAAe,CAAC;AAE9C,kDAAkD;AAClD,MAAM,WAAW,aAAc,SAAQ,KAAK;IAC1C,kDAAkD;IAClD,MAAM,EAAE,MAAM,EAAE,CAAC;CAClB;AAED;;;;;;;;GAQG;AACH,wBAAsB,MAAM,CAC1B,OAAO,EAAE,MAAM,EACf,OAAO,EAAE,OAAO,EAChB,QAAQ,EAAE,QAAQ,EAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GACjC,OAAO,CAAC,aAAa,EAAE,CAAC,CAc1B"}
|
package/dist/ingest.js
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Ingestion pipeline that chunks content and embeds each chunk.
|
|
3
|
-
*
|
|
4
|
-
* @module
|
|
5
|
-
*/
|
|
6
|
-
/**
|
|
7
|
-
* Chunk content and embed each chunk in a single pipeline.
|
|
8
|
-
*
|
|
9
|
-
* @param content - The raw text content to ingest.
|
|
10
|
-
* @param chunker - Splits the content into chunks.
|
|
11
|
-
* @param embedder - Produces embedding vectors for each chunk.
|
|
12
|
-
* @param metadata - Optional metadata passed through to the chunker.
|
|
13
|
-
* @returns An array of chunks with embedding vectors attached.
|
|
14
|
-
*/
|
|
15
|
-
export async function ingest(content, chunker, embedder, metadata) {
|
|
16
|
-
const chunks = chunker.chunk(content, metadata);
|
|
17
|
-
if (chunks.length === 0) {
|
|
18
|
-
return [];
|
|
19
|
-
}
|
|
20
|
-
const texts = chunks.map((c) => c.text);
|
|
21
|
-
const embeddings = await embedder.embedBatch(texts);
|
|
22
|
-
return chunks.map((chunk, i) => ({
|
|
23
|
-
...chunk,
|
|
24
|
-
vector: embeddings[i].vector,
|
|
25
|
-
}));
|
|
26
|
-
}
|
|
27
|
-
//# sourceMappingURL=ingest.js.map
|
package/dist/ingest.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"ingest.js","sourceRoot":"","sources":["../src/ingest.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAWH;;;;;;;;GAQG;AACH,MAAM,CAAC,KAAK,UAAU,MAAM,CAC1B,OAAe,EACf,OAAgB,EAChB,QAAkB,EAClB,QAAkC;IAElC,MAAM,MAAM,GAAY,OAAO,CAAC,KAAK,CAAC,OAAO,EAAE,QAAQ,CAAC,CAAC;IAEzD,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACxB,OAAO,EAAE,CAAC;IACZ,CAAC;IAED,MAAM,KAAK,GAAa,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAClD,MAAM,UAAU,GAAG,MAAM,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;IAEpD,OAAO,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC;QAC/B,GAAG,KAAK;QACR,MAAM,EAAE,UAAU,CAAC,CAAC,CAAC,CAAC,MAAM;KAC7B,CAAC,CAAC,CAAC;AACN,CAAC"}
|
package/dist/local-embedder.d.ts
DELETED
|
@@ -1,25 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Local ONNX-based embedder using `@huggingface/transformers`.
|
|
3
|
-
*
|
|
4
|
-
* Downloads and caches a HuggingFace model on first use, then runs
|
|
5
|
-
* inference locally on CPU — no API keys or external services required.
|
|
6
|
-
*
|
|
7
|
-
* @module
|
|
8
|
-
*/
|
|
9
|
-
import type { Embedder, EmbedderOptions } from "./embedder.js";
|
|
10
|
-
/**
|
|
11
|
-
* Create a local embedder that runs ONNX inference via `@huggingface/transformers`.
|
|
12
|
-
*
|
|
13
|
-
* The underlying pipeline is lazily initialized on the first call to
|
|
14
|
-
* {@link Embedder.embed | embed()} or {@link Embedder.embedBatch | embedBatch()}.
|
|
15
|
-
* This avoids blocking construction with a model download.
|
|
16
|
-
*
|
|
17
|
-
* When using the default model, dimensions are known upfront (384). When using
|
|
18
|
-
* a custom model without specifying dimensions, the value is inferred from the
|
|
19
|
-
* first embedding result.
|
|
20
|
-
*
|
|
21
|
-
* @param options - Optional configuration for model selection and dimensions.
|
|
22
|
-
* @returns An {@link Embedder} instance backed by local ONNX inference.
|
|
23
|
-
*/
|
|
24
|
-
export declare function createLocalEmbedder(options?: EmbedderOptions): Embedder;
|
|
25
|
-
//# sourceMappingURL=local-embedder.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"local-embedder.d.ts","sourceRoot":"","sources":["../src/local-embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAGH,OAAO,KAAK,EAAE,QAAQ,EAAE,eAAe,EAAmB,MAAM,eAAe,CAAC;AAQhF;;;;;;;;;;;;;GAaG;AACH,wBAAgB,mBAAmB,CAAC,OAAO,CAAC,EAAE,eAAe,GAAG,QAAQ,CA6DvE"}
|
package/dist/local-embedder.js
DELETED
|
@@ -1,88 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Local ONNX-based embedder using `@huggingface/transformers`.
|
|
3
|
-
*
|
|
4
|
-
* Downloads and caches a HuggingFace model on first use, then runs
|
|
5
|
-
* inference locally on CPU — no API keys or external services required.
|
|
6
|
-
*
|
|
7
|
-
* @module
|
|
8
|
-
*/
|
|
9
|
-
/** Default model: small, fast, good general-purpose English embeddings. */
|
|
10
|
-
const DEFAULT_MODEL_ID = "Xenova/all-MiniLM-L6-v2";
|
|
11
|
-
/** Default embedding dimensions for the default model. */
|
|
12
|
-
const DEFAULT_DIMENSIONS = 384;
|
|
13
|
-
/**
|
|
14
|
-
* Create a local embedder that runs ONNX inference via `@huggingface/transformers`.
|
|
15
|
-
*
|
|
16
|
-
* The underlying pipeline is lazily initialized on the first call to
|
|
17
|
-
* {@link Embedder.embed | embed()} or {@link Embedder.embedBatch | embedBatch()}.
|
|
18
|
-
* This avoids blocking construction with a model download.
|
|
19
|
-
*
|
|
20
|
-
* When using the default model, dimensions are known upfront (384). When using
|
|
21
|
-
* a custom model without specifying dimensions, the value is inferred from the
|
|
22
|
-
* first embedding result.
|
|
23
|
-
*
|
|
24
|
-
* @param options - Optional configuration for model selection and dimensions.
|
|
25
|
-
* @returns An {@link Embedder} instance backed by local ONNX inference.
|
|
26
|
-
*/
|
|
27
|
-
export function createLocalEmbedder(options) {
|
|
28
|
-
const modelId = options?.modelId ?? DEFAULT_MODEL_ID;
|
|
29
|
-
let resolvedDimensions = options?.dimensions ?? (modelId === DEFAULT_MODEL_ID ? DEFAULT_DIMENSIONS : 0);
|
|
30
|
-
let pipelinePromise;
|
|
31
|
-
/**
|
|
32
|
-
* Lazily initialise the feature-extraction pipeline.
|
|
33
|
-
* Concurrent calls share the same promise so the model is loaded once.
|
|
34
|
-
* If initialisation fails, the cached promise is cleared so future calls can retry.
|
|
35
|
-
*/
|
|
36
|
-
function getPipeline() {
|
|
37
|
-
if (!pipelinePromise) {
|
|
38
|
-
pipelinePromise = initPipeline(modelId).catch((error) => {
|
|
39
|
-
pipelinePromise = undefined;
|
|
40
|
-
throw error;
|
|
41
|
-
});
|
|
42
|
-
}
|
|
43
|
-
return pipelinePromise;
|
|
44
|
-
}
|
|
45
|
-
/** Run inference for a single text and return its vector. */
|
|
46
|
-
async function embedOne(pipe, text) {
|
|
47
|
-
const output = await pipe(text, { pooling: "mean", normalize: true });
|
|
48
|
-
const vector = Array.from(output.data);
|
|
49
|
-
if (resolvedDimensions === 0) {
|
|
50
|
-
resolvedDimensions = vector.length;
|
|
51
|
-
}
|
|
52
|
-
else if (vector.length !== resolvedDimensions) {
|
|
53
|
-
throw new Error(`Dimension mismatch: expected ${resolvedDimensions}, got ${vector.length}. ` +
|
|
54
|
-
`Check that the model "${modelId}" produces ${resolvedDimensions}-dim vectors.`);
|
|
55
|
-
}
|
|
56
|
-
return { text, vector };
|
|
57
|
-
}
|
|
58
|
-
return {
|
|
59
|
-
get dimensions() {
|
|
60
|
-
return resolvedDimensions;
|
|
61
|
-
},
|
|
62
|
-
async embed(text) {
|
|
63
|
-
const pipe = await getPipeline();
|
|
64
|
-
return embedOne(pipe, text);
|
|
65
|
-
},
|
|
66
|
-
async embedBatch(texts) {
|
|
67
|
-
if (texts.length === 0) {
|
|
68
|
-
return [];
|
|
69
|
-
}
|
|
70
|
-
const pipe = await getPipeline();
|
|
71
|
-
const results = [];
|
|
72
|
-
for (const text of texts) {
|
|
73
|
-
results.push(await embedOne(pipe, text));
|
|
74
|
-
}
|
|
75
|
-
return results;
|
|
76
|
-
},
|
|
77
|
-
};
|
|
78
|
-
}
|
|
79
|
-
/**
|
|
80
|
-
* Dynamically import `@huggingface/transformers` and create the pipeline.
|
|
81
|
-
*
|
|
82
|
-
* Uses dynamic import so the heavy ONNX runtime is only loaded when needed.
|
|
83
|
-
*/
|
|
84
|
-
async function initPipeline(modelId) {
|
|
85
|
-
const { pipeline } = await import("@huggingface/transformers");
|
|
86
|
-
return pipeline("feature-extraction", modelId, { dtype: "fp32" });
|
|
87
|
-
}
|
|
88
|
-
//# sourceMappingURL=local-embedder.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"local-embedder.js","sourceRoot":"","sources":["../src/local-embedder.ts"],"names":[],"mappings":"AAAA;;;;;;;GAOG;AAKH,2EAA2E;AAC3E,MAAM,gBAAgB,GAAW,yBAAyB,CAAC;AAE3D,0DAA0D;AAC1D,MAAM,kBAAkB,GAAW,GAAG,CAAC;AAEvC;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,mBAAmB,CAAC,OAAyB;IAC3D,MAAM,OAAO,GAAW,OAAO,EAAE,OAAO,IAAI,gBAAgB,CAAC;IAC7D,IAAI,kBAAkB,GACpB,OAAO,EAAE,UAAU,IAAI,CAAC,OAAO,KAAK,gBAAgB,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IAEjF,IAAI,eAA+D,CAAC;IAEpE;;;;OAIG;IACH,SAAS,WAAW;QAClB,IAAI,CAAC,eAAe,EAAE,CAAC;YACrB,eAAe,GAAG,YAAY,CAAC,OAAO,CAAC,CAAC,KAAK,CAAC,CAAC,KAAc,EAAE,EAAE;gBAC/D,eAAe,GAAG,SAAS,CAAC;gBAC5B,MAAM,KAAK,CAAC;YACd,CAAC,CAAC,CAAC;QACL,CAAC;QACD,OAAO,eAAe,CAAC;IACzB,CAAC;IAED,6DAA6D;IAC7D,KAAK,UAAU,QAAQ,CAAC,IAA+B,EAAE,IAAY;QACnE,MAAM,MAAM,GAAG,MAAM,IAAI,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QACtE,MAAM,MAAM,GAAa,KAAK,CAAC,IAAI,CAAC,MAAM,CAAC,IAAoB,CAAC,CAAC;QAEjE,IAAI,kBAAkB,KAAK,CAAC,EAAE,CAAC;YAC7B,kBAAkB,GAAG,MAAM,CAAC,MAAM,CAAC;QACrC,CAAC;aAAM,IAAI,MAAM,CAAC,MAAM,KAAK,kBAAkB,EAAE,CAAC;YAChD,MAAM,IAAI,KAAK,CACb,gCAAgC,kBAAkB,SAAS,MAAM,CAAC,MAAM,IAAI;gBAC5E,yBAAyB,OAAO,cAAc,kBAAkB,eAAe,CAChF,CAAC;QACJ,CAAC;QAED,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC;IAC1B,CAAC;IAED,OAAO;QACL,IAAI,UAAU;YACZ,OAAO,kBAAkB,CAAC;QAC5B,CAAC;QAED,KAAK,CAAC,KAAK,CAAC,IAAY;YACtB,MAAM,IAAI,GAAG,MAAM,WAAW,EAAE,CAAC;YACjC,OAAO,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;QAC9B,CAAC;QAED,KAAK,CAAC,UAAU,CAAC,KAAe;YAC9B,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACvB,OAAO,EAAE,CAAC;YACZ,CAAC;YACD,MAAM,IAAI,GAAG,MAAM,WAAW,EAAE,CAAC;YACjC,MAAM,OAAO,GAAsB,EAAE,CAAC;YACtC,KAAK,MAAM,IAAI,IAAI,KAAK,EAAE,CAAC;gBACzB,OAAO,CAAC,IAAI,CAAC,MAAM,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC,CAAC;YAC3C,CAAC;YACD,OAAO,OAAO,CAAC;QACjB,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,KAAK,UAAU,YAAY,CAAC,OAAe;IACzC,MAAM,EAAE,QAAQ,EAAE,GAAG,MAAM,MAAM,CAAC,2BAA2B,CAAC,CAAC;IAC/D,OAAO,QAAQ,CAAC,oBAAoB,EAAE,OAAO,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,CAAC;AACpE,CAAC"}
|
package/dist/logger.d.ts
DELETED
package/dist/logger.d.ts.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAa,EAAE,KAAK,MAAM,EAAE,MAAM,MAAM,CAAC;AAEzC,sDAAsD;AACtD,eAAO,MAAM,MAAM,EAAE,MAOnB,CAAC"}
|
package/dist/logger.js
DELETED
|
@@ -1,15 +0,0 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* Structured logger for the knowledge graph subsystem.
|
|
3
|
-
*
|
|
4
|
-
* @module
|
|
5
|
-
*/
|
|
6
|
-
import pino from "pino";
|
|
7
|
-
/** Pino logger instance for the knowledge package. */
|
|
8
|
-
export const logger = pino({
|
|
9
|
-
name: "grackle-knowledge",
|
|
10
|
-
level: process.env.LOG_LEVEL || "info",
|
|
11
|
-
transport: process.env.NODE_ENV !== "production"
|
|
12
|
-
? { target: "pino/file", options: { destination: 1 } }
|
|
13
|
-
: undefined,
|
|
14
|
-
});
|
|
15
|
-
//# sourceMappingURL=logger.js.map
|
package/dist/logger.js.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"logger.js","sourceRoot":"","sources":["../src/logger.ts"],"names":[],"mappings":"AAAA;;;;GAIG;AAEH,OAAO,IAAqB,MAAM,MAAM,CAAC;AAEzC,sDAAsD;AACtD,MAAM,CAAC,MAAM,MAAM,GAAW,IAAI,CAAC;IACjC,IAAI,EAAE,mBAAmB;IACzB,KAAK,EAAE,OAAO,CAAC,GAAG,CAAC,SAAS,IAAI,MAAM;IACtC,SAAS,EACP,OAAO,CAAC,GAAG,CAAC,QAAQ,KAAK,YAAY;QACnC,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,EAAE,OAAO,EAAE,EAAE,WAAW,EAAE,CAAC,EAAE,EAAE;QACtD,CAAC,CAAC,SAAS;CAChB,CAAC,CAAC"}
|