@pella-labs/pinakes 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (174) hide show
  1. package/README.md +208 -0
  2. package/dist/cli/audit.d.ts +30 -0
  3. package/dist/cli/audit.d.ts.map +1 -0
  4. package/dist/cli/audit.js +49 -0
  5. package/dist/cli/audit.js.map +1 -0
  6. package/dist/cli/export.d.ts +32 -0
  7. package/dist/cli/export.d.ts.map +1 -0
  8. package/dist/cli/export.js +73 -0
  9. package/dist/cli/export.js.map +1 -0
  10. package/dist/cli/import.d.ts +24 -0
  11. package/dist/cli/import.d.ts.map +1 -0
  12. package/dist/cli/import.js +96 -0
  13. package/dist/cli/import.js.map +1 -0
  14. package/dist/cli/index.d.ts +3 -0
  15. package/dist/cli/index.d.ts.map +1 -0
  16. package/dist/cli/index.js +172 -0
  17. package/dist/cli/index.js.map +1 -0
  18. package/dist/cli/purge.d.ts +23 -0
  19. package/dist/cli/purge.d.ts.map +1 -0
  20. package/dist/cli/purge.js +57 -0
  21. package/dist/cli/purge.js.map +1 -0
  22. package/dist/cli/rebuild.d.ts +54 -0
  23. package/dist/cli/rebuild.d.ts.map +1 -0
  24. package/dist/cli/rebuild.js +113 -0
  25. package/dist/cli/rebuild.js.map +1 -0
  26. package/dist/cli/serve.d.ts +49 -0
  27. package/dist/cli/serve.d.ts.map +1 -0
  28. package/dist/cli/serve.js +296 -0
  29. package/dist/cli/serve.js.map +1 -0
  30. package/dist/cli/status.d.ts +39 -0
  31. package/dist/cli/status.d.ts.map +1 -0
  32. package/dist/cli/status.js +108 -0
  33. package/dist/cli/status.js.map +1 -0
  34. package/dist/db/client.d.ts +109 -0
  35. package/dist/db/client.d.ts.map +1 -0
  36. package/dist/db/client.js +175 -0
  37. package/dist/db/client.js.map +1 -0
  38. package/dist/db/repository.d.ts +82 -0
  39. package/dist/db/repository.d.ts.map +1 -0
  40. package/dist/db/repository.js +173 -0
  41. package/dist/db/repository.js.map +1 -0
  42. package/dist/db/schema.d.ts +990 -0
  43. package/dist/db/schema.d.ts.map +1 -0
  44. package/dist/db/schema.js +259 -0
  45. package/dist/db/schema.js.map +1 -0
  46. package/dist/db/types.d.ts +28 -0
  47. package/dist/db/types.d.ts.map +1 -0
  48. package/dist/db/types.js +11 -0
  49. package/dist/db/types.js.map +1 -0
  50. package/dist/gaps/detector.d.ts +67 -0
  51. package/dist/gaps/detector.d.ts.map +1 -0
  52. package/dist/gaps/detector.js +160 -0
  53. package/dist/gaps/detector.js.map +1 -0
  54. package/dist/gate/budget.d.ts +90 -0
  55. package/dist/gate/budget.d.ts.map +1 -0
  56. package/dist/gate/budget.js +145 -0
  57. package/dist/gate/budget.js.map +1 -0
  58. package/dist/ingest/chokidar.d.ts +33 -0
  59. package/dist/ingest/chokidar.d.ts.map +1 -0
  60. package/dist/ingest/chokidar.js +152 -0
  61. package/dist/ingest/chokidar.js.map +1 -0
  62. package/dist/ingest/ingester.d.ts +117 -0
  63. package/dist/ingest/ingester.d.ts.map +1 -0
  64. package/dist/ingest/ingester.js +312 -0
  65. package/dist/ingest/ingester.js.map +1 -0
  66. package/dist/ingest/manifest.d.ts +87 -0
  67. package/dist/ingest/manifest.d.ts.map +1 -0
  68. package/dist/ingest/manifest.js +223 -0
  69. package/dist/ingest/manifest.js.map +1 -0
  70. package/dist/ingest/memory-store.d.ts +55 -0
  71. package/dist/ingest/memory-store.d.ts.map +1 -0
  72. package/dist/ingest/memory-store.js +94 -0
  73. package/dist/ingest/memory-store.js.map +1 -0
  74. package/dist/ingest/parse/chunk.d.ts +15 -0
  75. package/dist/ingest/parse/chunk.d.ts.map +1 -0
  76. package/dist/ingest/parse/chunk.js +88 -0
  77. package/dist/ingest/parse/chunk.js.map +1 -0
  78. package/dist/ingest/parse/markdown.d.ts +64 -0
  79. package/dist/ingest/parse/markdown.d.ts.map +1 -0
  80. package/dist/ingest/parse/markdown.js +152 -0
  81. package/dist/ingest/parse/markdown.js.map +1 -0
  82. package/dist/ingest/queue.d.ts +21 -0
  83. package/dist/ingest/queue.d.ts.map +1 -0
  84. package/dist/ingest/queue.js +24 -0
  85. package/dist/ingest/queue.js.map +1 -0
  86. package/dist/ingest/source.d.ts +42 -0
  87. package/dist/ingest/source.d.ts.map +1 -0
  88. package/dist/ingest/source.js +19 -0
  89. package/dist/ingest/source.js.map +1 -0
  90. package/dist/mcp/envelope.d.ts +73 -0
  91. package/dist/mcp/envelope.d.ts.map +1 -0
  92. package/dist/mcp/envelope.js +46 -0
  93. package/dist/mcp/envelope.js.map +1 -0
  94. package/dist/mcp/tools/execute.d.ts +55 -0
  95. package/dist/mcp/tools/execute.d.ts.map +1 -0
  96. package/dist/mcp/tools/execute.js +232 -0
  97. package/dist/mcp/tools/execute.js.map +1 -0
  98. package/dist/mcp/tools/search.d.ts +53 -0
  99. package/dist/mcp/tools/search.d.ts.map +1 -0
  100. package/dist/mcp/tools/search.js +114 -0
  101. package/dist/mcp/tools/search.js.map +1 -0
  102. package/dist/observability/audit.d.ts +25 -0
  103. package/dist/observability/audit.d.ts.map +1 -0
  104. package/dist/observability/audit.js +38 -0
  105. package/dist/observability/audit.js.map +1 -0
  106. package/dist/observability/logger.d.ts +4 -0
  107. package/dist/observability/logger.d.ts.map +1 -0
  108. package/dist/observability/logger.js +56 -0
  109. package/dist/observability/logger.js.map +1 -0
  110. package/dist/observability/metrics.d.ts +38 -0
  111. package/dist/observability/metrics.d.ts.map +1 -0
  112. package/dist/observability/metrics.js +64 -0
  113. package/dist/observability/metrics.js.map +1 -0
  114. package/dist/retrieval/embedder.d.ts +130 -0
  115. package/dist/retrieval/embedder.d.ts.map +1 -0
  116. package/dist/retrieval/embedder.js +278 -0
  117. package/dist/retrieval/embedder.js.map +1 -0
  118. package/dist/retrieval/fts.d.ts +42 -0
  119. package/dist/retrieval/fts.d.ts.map +1 -0
  120. package/dist/retrieval/fts.js +46 -0
  121. package/dist/retrieval/fts.js.map +1 -0
  122. package/dist/retrieval/hybrid.d.ts +43 -0
  123. package/dist/retrieval/hybrid.d.ts.map +1 -0
  124. package/dist/retrieval/hybrid.js +120 -0
  125. package/dist/retrieval/hybrid.js.map +1 -0
  126. package/dist/retrieval/vec.d.ts +39 -0
  127. package/dist/retrieval/vec.d.ts.map +1 -0
  128. package/dist/retrieval/vec.js +50 -0
  129. package/dist/retrieval/vec.js.map +1 -0
  130. package/dist/sandbox/bindings/budget.d.ts +10 -0
  131. package/dist/sandbox/bindings/budget.d.ts.map +1 -0
  132. package/dist/sandbox/bindings/budget.js +44 -0
  133. package/dist/sandbox/bindings/budget.js.map +1 -0
  134. package/dist/sandbox/bindings/install.d.ts +23 -0
  135. package/dist/sandbox/bindings/install.d.ts.map +1 -0
  136. package/dist/sandbox/bindings/install.js +15 -0
  137. package/dist/sandbox/bindings/install.js.map +1 -0
  138. package/dist/sandbox/bindings/kg.d.ts +29 -0
  139. package/dist/sandbox/bindings/kg.d.ts.map +1 -0
  140. package/dist/sandbox/bindings/kg.js +323 -0
  141. package/dist/sandbox/bindings/kg.js.map +1 -0
  142. package/dist/sandbox/bindings/logger.d.ts +11 -0
  143. package/dist/sandbox/bindings/logger.d.ts.map +1 -0
  144. package/dist/sandbox/bindings/logger.js +33 -0
  145. package/dist/sandbox/bindings/logger.js.map +1 -0
  146. package/dist/sandbox/bindings/write.d.ts +34 -0
  147. package/dist/sandbox/bindings/write.d.ts.map +1 -0
  148. package/dist/sandbox/bindings/write.js +195 -0
  149. package/dist/sandbox/bindings/write.js.map +1 -0
  150. package/dist/sandbox/executor.d.ts +68 -0
  151. package/dist/sandbox/executor.d.ts.map +1 -0
  152. package/dist/sandbox/executor.js +280 -0
  153. package/dist/sandbox/executor.js.map +1 -0
  154. package/dist/sandbox/helpers.d.ts +26 -0
  155. package/dist/sandbox/helpers.d.ts.map +1 -0
  156. package/dist/sandbox/helpers.js +131 -0
  157. package/dist/sandbox/helpers.js.map +1 -0
  158. package/dist/sandbox/pool.d.ts +63 -0
  159. package/dist/sandbox/pool.d.ts.map +1 -0
  160. package/dist/sandbox/pool.js +98 -0
  161. package/dist/sandbox/pool.js.map +1 -0
  162. package/dist/sandbox/vendored-codemode.d.ts +99 -0
  163. package/dist/sandbox/vendored-codemode.d.ts.map +1 -0
  164. package/dist/sandbox/vendored-codemode.js +471 -0
  165. package/dist/sandbox/vendored-codemode.js.map +1 -0
  166. package/dist/server.d.ts +3 -0
  167. package/dist/server.d.ts.map +1 -0
  168. package/dist/server.js +74 -0
  169. package/dist/server.js.map +1 -0
  170. package/dist/spike.d.ts +15 -0
  171. package/dist/spike.d.ts.map +1 -0
  172. package/dist/spike.js +90 -0
  173. package/dist/spike.js.map +1 -0
  174. package/package.json +60 -0
@@ -0,0 +1,38 @@
1
+ import { appendFileSync, existsSync, mkdirSync } from 'node:fs';
2
+ import { dirname } from 'node:path';
3
+ import { logger } from './logger.js';
4
+ /**
5
+ * Write an audit row to the appropriate DB and mirror to JSONL.
6
+ *
7
+ * @param writer The DB writer for the scope-appropriate bundle.
8
+ * @param jsonlPath The path for the JSONL mirror file.
9
+ * @param entry Audit entry data.
10
+ */
11
+ export function writeAuditRow(writer, jsonlPath, entry) {
12
+ const ts = Date.now();
13
+ // Write to kg_audit table
14
+ try {
15
+ writer
16
+ .prepare(`INSERT INTO kg_audit (ts, tool_name, scope_requested, caller_ctx, response_tokens, error)
17
+ VALUES (?, ?, ?, ?, ?, ?)`)
18
+ .run(ts, entry.toolName, entry.scopeRequested, entry.callerCtx ?? null, entry.responseTokens ?? null, entry.error ?? null);
19
+ }
20
+ catch (err) {
21
+ logger.warn({ err, entry }, 'failed to write kg_audit row');
22
+ }
23
+ // Mirror to JSONL
24
+ if (jsonlPath) {
25
+ try {
26
+ const dir = dirname(jsonlPath);
27
+ if (!existsSync(dir)) {
28
+ mkdirSync(dir, { recursive: true });
29
+ }
30
+ const line = JSON.stringify({ ts, ...entry }) + '\n';
31
+ appendFileSync(jsonlPath, line, 'utf-8');
32
+ }
33
+ catch (err) {
34
+ logger.warn({ err, jsonlPath }, 'failed to append audit JSONL');
35
+ }
36
+ }
37
+ }
38
+ //# sourceMappingURL=audit.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"audit.js","sourceRoot":"","sources":["../../src/observability/audit.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,cAAc,EAAE,UAAU,EAAE,SAAS,EAAE,MAAM,SAAS,CAAC;AAChE,OAAO,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AAGpC,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAmBrC;;;;;;GAMG;AACH,MAAM,UAAU,aAAa,CAC3B,MAA4B,EAC5B,SAA6B,EAC7B,KAAiB;IAEjB,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAEtB,0BAA0B;IAC1B,IAAI,CAAC;QACH,MAAM;aACH,OAAO,CACN;mCAC2B,CAC5B;aACA,GAAG,CACF,EAAE,EACF,KAAK,CAAC,QAAQ,EACd,KAAK,CAAC,cAAc,EACpB,KAAK,CAAC,SAAS,IAAI,IAAI,EACvB,KAAK,CAAC,cAAc,IAAI,IAAI,EAC5B,KAAK,CAAC,KAAK,IAAI,IAAI,CACpB,CAAC;IACN,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,EAAE,8BAA8B,CAAC,CAAC;IAC9D,CAAC;IAED,kBAAkB;IAClB,IAAI,SAAS,EAAE,CAAC;QACd,IAAI,CAAC;YACH,MAAM,GAAG,GAAG,OAAO,CAAC,SAAS,CAAC,CAAC;YAC/B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACrB,SAAS,CAAC,GAAG,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;YACtC,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,SAAS,CAAC,EAAE,EAAE,EAAE,GAAG,KAAK,EAAE,CAAC,GAAG,IAAI,CAAC;YACrD,cAAc,CAAC,SAAS,EAAE,IAAI,EAAE,OAAO,CAAC,CAAC;QAC3C,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,SAAS,EAAE,EAAE,8BAA8B,CAAC,CAAC;QAClE,CAAC;IACH,CAAC;AACH,CAAC"}
@@ -0,0 +1,4 @@
1
+ import { type Logger } from 'pino';
2
+ export declare const logger: Logger;
3
+ export declare function child(bindings: Record<string, unknown>): Logger;
4
+ //# sourceMappingURL=logger.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"logger.d.ts","sourceRoot":"","sources":["../../src/observability/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAQ,KAAK,MAAM,EAAsB,MAAM,MAAM,CAAC;AAwD7D,eAAO,MAAM,MAAM,EAAE,MAAuB,CAAC;AAE7C,wBAAgB,KAAK,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,GAAG,MAAM,CAE/D"}
@@ -0,0 +1,56 @@
1
+ import { pino } from 'pino';
2
+ /**
3
+ * Pino logger for KG-MCP.
4
+ *
5
+ * CRITICAL: MCP stdio transport uses stdout for the JSON-RPC protocol. Every
6
+ * log line must go to stderr — writing to stdout would corrupt the protocol
7
+ * stream and crash the client. Pino's default `destination()` is already
8
+ * stderr when we pass `process.stderr.fd`, and we never pass a custom
9
+ * destination that would route to stdout.
10
+ *
11
+ * Pretty transport is only enabled when:
12
+ * 1. `KG_LOG_LEVEL=debug` or `trace`, AND
13
+ * 2. stderr is a TTY (i.e. a developer is watching in a terminal)
14
+ *
15
+ * In production (Pharos-spawned stdio child, non-TTY), logs stay as newline-
16
+ * delimited JSON for machine consumption.
17
+ */
18
+ const level = process.env.KG_LOG_LEVEL ?? 'info';
19
+ const isVerbose = level === 'debug' || level === 'trace';
20
+ const isTty = process.stderr.isTTY ?? false;
21
+ const usePretty = isVerbose && isTty;
22
+ const baseOptions = {
23
+ level,
24
+ base: {
25
+ service: 'kg-mcp',
26
+ pid: process.pid,
27
+ },
28
+ timestamp: pino.stdTimeFunctions.isoTime,
29
+ };
30
+ function createLogger() {
31
+ if (usePretty) {
32
+ // With a transport, pino offloads to a worker; the transport target itself
33
+ // writes to stderr via `destination: 2`.
34
+ return pino({
35
+ ...baseOptions,
36
+ transport: {
37
+ target: 'pino-pretty',
38
+ options: {
39
+ colorize: true,
40
+ destination: 2, // stderr
41
+ translateTime: 'HH:MM:ss.l',
42
+ ignore: 'pid,hostname,service',
43
+ },
44
+ },
45
+ });
46
+ }
47
+ // Default path: write NDJSON directly to stderr so stdout stays clean for
48
+ // the MCP JSON-RPC protocol. `process.stderr` satisfies pino's
49
+ // DestinationStream contract.
50
+ return pino(baseOptions, process.stderr);
51
+ }
52
+ export const logger = createLogger();
53
+ export function child(bindings) {
54
+ return logger.child(bindings);
55
+ }
56
+ //# sourceMappingURL=logger.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"logger.js","sourceRoot":"","sources":["../../src/observability/logger.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,IAAI,EAAmC,MAAM,MAAM,CAAC;AAE7D;;;;;;;;;;;;;;;GAeG;AAEH,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,YAAY,IAAI,MAAM,CAAC;AACjD,MAAM,SAAS,GAAG,KAAK,KAAK,OAAO,IAAI,KAAK,KAAK,OAAO,CAAC;AACzD,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,IAAI,KAAK,CAAC;AAC5C,MAAM,SAAS,GAAG,SAAS,IAAI,KAAK,CAAC;AAErC,MAAM,WAAW,GAAkB;IACjC,KAAK;IACL,IAAI,EAAE;QACJ,OAAO,EAAE,QAAQ;QACjB,GAAG,EAAE,OAAO,CAAC,GAAG;KACjB;IACD,SAAS,EAAE,IAAI,CAAC,gBAAgB,CAAC,OAAO;CACzC,CAAC;AAEF,SAAS,YAAY;IACnB,IAAI,SAAS,EAAE,CAAC;QACd,2EAA2E;QAC3E,yCAAyC;QACzC,OAAO,IAAI,CAAC;YACV,GAAG,WAAW;YACd,SAAS,EAAE;gBACT,MAAM,EAAE,aAAa;gBACrB,OAAO,EAAE;oBACP,QAAQ,EAAE,IAAI;oBACd,WAAW,EAAE,CAAC,EAAE,SAAS;oBACzB,aAAa,EAAE,YAAY;oBAC3B,MAAM,EAAE,sBAAsB;iBAC/B;aACF;SACF,CAAC,CAAC;IACL,CAAC;IACD,0EAA0E;IAC1E,+DAA+D;IAC/D,8BAA8B;IAC9B,OAAO,IAAI,CAAC,WAAW,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;AAC3C,CAAC;AAED,MAAM,CAAC,MAAM,MAAM,GAAW,YAAY,EAAE,CAAC;AAE7C,MAAM,UAAU,KAAK,CAAC,QAAiC;IACrD,OAAO,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;AAChC,CAAC"}
@@ -0,0 +1,38 @@
1
+ /**
2
+ * In-process metrics counters with SIGHUP dump.
3
+ *
4
+ * CLAUDE.md §Phase 7: "Metrics dump on SIGHUP: emit all counters as a
5
+ * single JSON line to stderr."
6
+ *
7
+ * All counters are monotonic (never reset). The SIGHUP handler snapshots
8
+ * everything and writes it to stderr as NDJSON so it doesn't collide with
9
+ * the MCP stdio protocol on stdout.
10
+ */
11
+ export interface LatencyBucket {
12
+ count: number;
13
+ sum: number;
14
+ max: number;
15
+ }
16
+ export interface MetricsSnapshot {
17
+ tool_calls: Record<string, number>;
18
+ tool_errors: Record<string, number>;
19
+ tool_latency_ms: Record<string, LatencyBucket>;
20
+ ingest_files: number;
21
+ ingest_errors: number;
22
+ uptime_s: number;
23
+ }
24
+ declare class Metrics {
25
+ private toolCalls;
26
+ private toolErrors;
27
+ private toolLatency;
28
+ private ingestFiles;
29
+ private ingestErrors;
30
+ private readonly startedAt;
31
+ recordToolCall(tool: string, latencyMs: number, error?: boolean): void;
32
+ recordIngest(error?: boolean): void;
33
+ snapshot(): MetricsSnapshot;
34
+ }
35
+ export declare const metrics: Metrics;
36
+ export declare function installSighupHandler(): void;
37
+ export {};
38
+ //# sourceMappingURL=metrics.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"metrics.d.ts","sourceRoot":"","sources":["../../src/observability/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAIH,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,GAAG,EAAE,MAAM,CAAC;IACZ,GAAG,EAAE,MAAM,CAAC;CACb;AAED,MAAM,WAAW,eAAe;IAC9B,UAAU,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACnC,WAAW,EAAE,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IACpC,eAAe,EAAE,MAAM,CAAC,MAAM,EAAE,aAAa,CAAC,CAAC;IAC/C,YAAY,EAAE,MAAM,CAAC;IACrB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,CAAC;CAClB;AAED,cAAM,OAAO;IACX,OAAO,CAAC,SAAS,CAA6B;IAC9C,OAAO,CAAC,UAAU,CAA6B;IAC/C,OAAO,CAAC,WAAW,CAAoC;IACvD,OAAO,CAAC,WAAW,CAAK;IACxB,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAc;IAExC,cAAc,CAAC,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,OAAO,GAAG,IAAI;IAYtE,YAAY,CAAC,KAAK,CAAC,EAAE,OAAO,GAAG,IAAI;IAKnC,QAAQ,IAAI,eAAe;CAU5B;AAED,eAAO,MAAM,OAAO,SAAgB,CAAC;AAOrC,wBAAgB,oBAAoB,IAAI,IAAI,CAS3C"}
@@ -0,0 +1,64 @@
1
+ /**
2
+ * In-process metrics counters with SIGHUP dump.
3
+ *
4
+ * CLAUDE.md §Phase 7: "Metrics dump on SIGHUP: emit all counters as a
5
+ * single JSON line to stderr."
6
+ *
7
+ * All counters are monotonic (never reset). The SIGHUP handler snapshots
8
+ * everything and writes it to stderr as NDJSON so it doesn't collide with
9
+ * the MCP stdio protocol on stdout.
10
+ */
11
+ import { logger } from './logger.js';
12
+ class Metrics {
13
+ toolCalls = new Map();
14
+ toolErrors = new Map();
15
+ toolLatency = new Map();
16
+ ingestFiles = 0;
17
+ ingestErrors = 0;
18
+ startedAt = Date.now();
19
+ recordToolCall(tool, latencyMs, error) {
20
+ this.toolCalls.set(tool, (this.toolCalls.get(tool) ?? 0) + 1);
21
+ if (error) {
22
+ this.toolErrors.set(tool, (this.toolErrors.get(tool) ?? 0) + 1);
23
+ }
24
+ const lat = this.toolLatency.get(tool) ?? { count: 0, sum: 0, max: 0 };
25
+ lat.count++;
26
+ lat.sum += latencyMs;
27
+ lat.max = Math.max(lat.max, latencyMs);
28
+ this.toolLatency.set(tool, lat);
29
+ }
30
+ recordIngest(error) {
31
+ if (error)
32
+ this.ingestErrors++;
33
+ else
34
+ this.ingestFiles++;
35
+ }
36
+ snapshot() {
37
+ return {
38
+ tool_calls: Object.fromEntries(this.toolCalls),
39
+ tool_errors: Object.fromEntries(this.toolErrors),
40
+ tool_latency_ms: Object.fromEntries(this.toolLatency),
41
+ ingest_files: this.ingestFiles,
42
+ ingest_errors: this.ingestErrors,
43
+ uptime_s: Math.round((Date.now() - this.startedAt) / 1000),
44
+ };
45
+ }
46
+ }
47
+ export const metrics = new Metrics();
48
+ /**
49
+ * Install SIGHUP handler to dump metrics as a single JSON line to stderr.
50
+ * Safe to call multiple times — only installs once.
51
+ */
52
+ let sighupInstalled = false;
53
+ export function installSighupHandler() {
54
+ if (sighupInstalled)
55
+ return;
56
+ sighupInstalled = true;
57
+ process.on('SIGHUP', () => {
58
+ const snap = metrics.snapshot();
59
+ // Write directly to stderr to avoid pino's async buffering.
60
+ process.stderr.write(JSON.stringify({ kg_mcp_metrics: snap }) + '\n');
61
+ logger.info('SIGHUP: metrics dumped to stderr');
62
+ });
63
+ }
64
+ //# sourceMappingURL=metrics.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"metrics.js","sourceRoot":"","sources":["../../src/observability/metrics.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AAEH,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAC;AAiBrC,MAAM,OAAO;IACH,SAAS,GAAG,IAAI,GAAG,EAAkB,CAAC;IACtC,UAAU,GAAG,IAAI,GAAG,EAAkB,CAAC;IACvC,WAAW,GAAG,IAAI,GAAG,EAAyB,CAAC;IAC/C,WAAW,GAAG,CAAC,CAAC;IAChB,YAAY,GAAG,CAAC,CAAC;IACR,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAExC,cAAc,CAAC,IAAY,EAAE,SAAiB,EAAE,KAAe;QAC7D,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAC9D,IAAI,KAAK,EAAE,CAAC;YACV,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC;QAClE,CAAC;QACD,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC;QACvE,GAAG,CAAC,KAAK,EAAE,CAAC;QACZ,GAAG,CAAC,GAAG,IAAI,SAAS,CAAC;QACrB,GAAG,CAAC,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,EAAE,SAAS,CAAC,CAAC;QACvC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IAClC,CAAC;IAED,YAAY,CAAC,KAAe;QAC1B,IAAI,KAAK;YAAE,IAAI,CAAC,YAAY,EAAE,CAAC;;YAC1B,IAAI,CAAC,WAAW,EAAE,CAAC;IAC1B,CAAC;IAED,QAAQ;QACN,OAAO;YACL,UAAU,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,SAAS,CAAC;YAC9C,WAAW,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,UAAU,CAAC;YAChD,eAAe,EAAE,MAAM,CAAC,WAAW,CAAC,IAAI,CAAC,WAAW,CAAC;YACrD,YAAY,EAAE,IAAI,CAAC,WAAW;YAC9B,aAAa,EAAE,IAAI,CAAC,YAAY;YAChC,QAAQ,EAAE,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC;SAC3D,CAAC;IACJ,CAAC;CACF;AAED,MAAM,CAAC,MAAM,OAAO,GAAG,IAAI,OAAO,EAAE,CAAC;AAErC;;;GAGG;AACH,IAAI,eAAe,GAAG,KAAK,CAAC;AAC5B,MAAM,UAAU,oBAAoB;IAClC,IAAI,eAAe;QAAE,OAAO;IAC5B,eAAe,GAAG,IAAI,CAAC;IACvB,OAAO,CAAC,EAAE,CAAC,QAAQ,EAAE,GAAG,EAAE;QACxB,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,EAAE,CAAC;QAChC,4DAA4D;QAC5D,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,EAAE,cAAc,EAAE,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;QACtE,MAAM,CAAC,IAAI,CAAC,kCAAkC,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;AACL,CAAC"}
@@ -0,0 +1,130 @@
1
+ /**
2
+ * Embedder interface + default `@xenova/transformers` MiniLM implementation.
3
+ *
4
+ * **Why this lives in `retrieval/`** (not `ingest/`): the embedder is shared
5
+ * between the ingest path (computing chunk embeddings on insert) and the
6
+ * future query path in Phase 4 (computing query embeddings for vector
7
+ * search). Putting it under `retrieval/` keeps the code-mode bindings'
8
+ * mental model consistent — `kg.vec()` and ingest both call into the same
9
+ * factory.
10
+ *
11
+ * **Provider strategy** (CLAUDE.md §AI Rules #3): Phase 2 ships only the
12
+ * bundled MiniLM provider. Phase 4 adds the env-driven factory:
13
+ * - `KG_EMBED_PROVIDER=transformers` (default, this file)
14
+ * - `KG_EMBED_PROVIDER=ollama` (HTTP, user-controlled)
15
+ * - `KG_EMBED_PROVIDER=voyage` (HTTPS, paid)
16
+ * - `KG_EMBED_PROVIDER=openai` (HTTPS, paid)
17
+ *
18
+ * **Failure mode** (CLAUDE.md §AI Rules #4): if the embedder fails during
19
+ * ingest, the ingester logs a warning and inserts the node + chunks WITHOUT
20
+ * vec rows. Query-time degrades gracefully to FTS5-only for affected chunks.
21
+ * The contract here is that `embed()` may throw — callers must catch.
22
+ *
23
+ * **Singleton**: model load is ~25MB and takes ~800ms cold. We load once
24
+ * per process via `getDefaultEmbedder()` and share the instance. The
25
+ * `warmup()` method preloads the model proactively at server startup.
26
+ */
27
+ /** Embedding output dimension for MiniLM. Locked — changing requires a vec table rebuild. */
28
+ export declare const EMBEDDING_DIM = 384;
29
+ /**
30
+ * Generic embedder contract. Implementations: `TransformersEmbedder` (default),
31
+ * `CountingEmbedder` (test wrapper), and Phase 4's `OllamaEmbedder` /
32
+ * `VoyageEmbedder` / `OpenAIEmbedder`.
33
+ */
34
+ export interface Embedder {
35
+ /** Output dimension — load-bearing for the `kg_chunks_vec` schema */
36
+ readonly dim: number;
37
+ /** Eagerly load the underlying model. Optional — embed() will lazy-load on first call. */
38
+ warmup(): Promise<void>;
39
+ /** Embed a single text. Returns a Float32Array of length `dim`. */
40
+ embed(text: string): Promise<Float32Array>;
41
+ }
42
+ /**
43
+ * MiniLM via @xenova/transformers, running locally in-process. No network
44
+ * calls after first model download. The model is cached under
45
+ * `~/.cache/huggingface/` (or `XENOVA_CACHE_DIR` if set) so subsequent
46
+ * starts are fast (<500ms).
47
+ *
48
+ * Output: 384-dim Float32Array, mean-pooled across tokens, L2-normalized.
49
+ * The `pooling: 'mean'` and `normalize: true` options match the upstream
50
+ * Sentence Transformers reference inference, so embeddings produced here
51
+ * are interchangeable with embeddings produced by the Python lib.
52
+ */
53
+ export declare class TransformersEmbedder implements Embedder {
54
+ private readonly modelName;
55
+ readonly dim = 384;
56
+ private pipelinePromise;
57
+ constructor(modelName?: string);
58
+ warmup(): Promise<void>;
59
+ embed(text: string): Promise<Float32Array>;
60
+ private getPipeline;
61
+ }
62
+ /**
63
+ * Test-only wrapper that delegates to a real embedder and counts the number
64
+ * of `embed()` calls. Used by `ingester.test.ts` to verify that re-ingesting
65
+ * a file with one mutated paragraph triggers exactly one new embedder call —
66
+ * the load-bearing per-chunk skip-unchanged optimization (Loop 6.5 A4).
67
+ *
68
+ * Lives next to the production embedder (not under `__tests__/`) so other
69
+ * test files can import it without circular dep issues.
70
+ */
71
+ export declare class CountingEmbedder implements Embedder {
72
+ private readonly inner;
73
+ readonly dim: number;
74
+ /** Number of times `embed()` has been called since construction (or `reset()`). */
75
+ calls: number;
76
+ constructor(inner: Embedder);
77
+ warmup(): Promise<void>;
78
+ embed(text: string): Promise<Float32Array>;
79
+ reset(): void;
80
+ }
81
+ /**
82
+ * Ollama embedder via HTTP POST to `/api/embeddings`.
83
+ * Requires `KG_OLLAMA_URL` and `KG_OLLAMA_MODEL` env vars.
84
+ */
85
+ export declare class OllamaEmbedder implements Embedder {
86
+ private readonly baseUrl;
87
+ private readonly model;
88
+ readonly dim: number;
89
+ constructor(baseUrl: string, model: string, dim?: number);
90
+ warmup(): Promise<void>;
91
+ embed(text: string): Promise<Float32Array>;
92
+ }
93
+ /**
94
+ * Voyage AI embedder via HTTPS POST to `https://api.voyageai.com/v1/embeddings`.
95
+ * Requires `KG_VOYAGE_API_KEY` env var.
96
+ */
97
+ export declare class VoyageEmbedder implements Embedder {
98
+ private readonly apiKey;
99
+ private readonly model;
100
+ readonly dim: number;
101
+ constructor(apiKey: string, model?: string, dim?: number);
102
+ warmup(): Promise<void>;
103
+ embed(text: string): Promise<Float32Array>;
104
+ }
105
+ /**
106
+ * OpenAI embedder via HTTPS POST to `https://api.openai.com/v1/embeddings`.
107
+ * Requires `KG_OPENAI_API_KEY` env var.
108
+ */
109
+ export declare class OpenAIEmbedder implements Embedder {
110
+ private readonly apiKey;
111
+ private readonly model;
112
+ readonly dim: number;
113
+ constructor(apiKey: string, model?: string, dim?: number);
114
+ warmup(): Promise<void>;
115
+ embed(text: string): Promise<Float32Array>;
116
+ }
117
+ export type EmbedProvider = 'transformers' | 'ollama' | 'voyage' | 'openai';
118
+ /**
119
+ * Create an embedder instance based on the provider name and environment.
120
+ * Used at server startup. Falls back to TransformersEmbedder if the
121
+ * provider is unrecognized.
122
+ */
123
+ export declare function createEmbedder(provider?: EmbedProvider): Embedder;
124
+ /**
125
+ * Singleton accessor. Always returns the same `TransformersEmbedder` instance
126
+ * within a process so the model loads exactly once. Tests that need a fresh
127
+ * instance can construct `new TransformersEmbedder()` directly.
128
+ */
129
+ export declare function getDefaultEmbedder(): TransformersEmbedder;
130
+ //# sourceMappingURL=embedder.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"embedder.d.ts","sourceRoot":"","sources":["../../src/retrieval/embedder.ts"],"names":[],"mappings":"AAIA;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,6FAA6F;AAC7F,eAAO,MAAM,aAAa,MAAM,CAAC;AAEjC;;;;GAIG;AACH,MAAM,WAAW,QAAQ;IACvB,qEAAqE;IACrE,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,0FAA0F;IAC1F,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;IACxB,mEAAmE;IACnE,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC,CAAC;CAC5C;AAQD;;;;;;;;;;GAUG;AACH,qBAAa,oBAAqB,YAAW,QAAQ;IAKvC,OAAO,CAAC,QAAQ,CAAC,SAAS;IAJtC,QAAQ,CAAC,GAAG,OAAiB;IAE7B,OAAO,CAAC,eAAe,CAAmD;gBAE7C,SAAS,GAAE,MAAsB;IAOxD,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAIvB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;YAclC,WAAW;CAmB1B;AAMD;;;;;;;;GAQG;AACH,qBAAa,gBAAiB,YAAW,QAAQ;IAKnC,OAAO,CAAC,QAAQ,CAAC,KAAK;IAJlC,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;IACrB,mFAAmF;IACnF,KAAK,SAAK;gBAEmB,KAAK,EAAE,QAAQ;IAItC,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAIvB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;IAKhD,KAAK,IAAI,IAAI;CAGd;AAMD;;;GAGG;AACH,qBAAa,cAAe,YAAW,QAAQ;IAI3C,OAAO,CAAC,QAAQ,CAAC,OAAO;IACxB,OAAO,CAAC,QAAQ,CAAC,KAAK;IAJxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;gBAGF,OAAO,EAAE,MAAM,EACf,KAAK,EAAE,MAAM,EAC9B,GAAG,GAAE,MAAsB;IAKvB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAIvB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;CAkBjD;AAMD;;;GAGG;AACH,qBAAa,cAAe,YAAW,QAAQ;IAI3C,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,KAAK;IAJxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;gBAGF,MAAM,EAAE,MAAM,EACd,KAAK,GAAE,MAAwB,EAChD,GAAG,GAAE,MAAsB;IAKvB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;CAqBjD;AAMD;;;GAGG;AACH,qBAAa,cAAe,YAAW,QAAQ;IAI3C,OAAO,CAAC,QAAQ,CAAC,MAAM;IACvB,OAAO,CAAC,QAAQ,CAAC,KAAK;IAJxB,QAAQ,CAAC,GAAG,EAAE,MAAM,CAAC;gBAGF,MAAM,EAAE,MAAM,EACd,KAAK,GAAE,MAAiC,EACzD,GAAG,GAAE,MAAsB;IAKvB,MAAM,IAAI,OAAO,CAAC,IAAI,CAAC;IAEvB,KAAK,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,YAAY,CAAC;CAqBjD;AAMD,MAAM,MAAM,aAAa,GAAG,cAAc,GAAG,QAAQ,GAAG,QAAQ,GAAG,QAAQ,CAAC;AAE5E;;;;GAIG;AACH,wBAAgB,cAAc,CAAC,QAAQ,CAAC,EAAE,aAAa,GAAG,QAAQ,CAyBjE;AAQD;;;;GAIG;AACH,wBAAgB,kBAAkB,IAAI,oBAAoB,CAKzD"}
@@ -0,0 +1,278 @@
1
+ import { pipeline, env } from '@xenova/transformers';
2
+ import { logger } from '../observability/logger.js';
3
+ /**
4
+ * Embedder interface + default `@xenova/transformers` MiniLM implementation.
5
+ *
6
+ * **Why this lives in `retrieval/`** (not `ingest/`): the embedder is shared
7
+ * between the ingest path (computing chunk embeddings on insert) and the
8
+ * future query path in Phase 4 (computing query embeddings for vector
9
+ * search). Putting it under `retrieval/` keeps the code-mode bindings'
10
+ * mental model consistent — `kg.vec()` and ingest both call into the same
11
+ * factory.
12
+ *
13
+ * **Provider strategy** (CLAUDE.md §AI Rules #3): Phase 2 ships only the
14
+ * bundled MiniLM provider. Phase 4 adds the env-driven factory:
15
+ * - `KG_EMBED_PROVIDER=transformers` (default, this file)
16
+ * - `KG_EMBED_PROVIDER=ollama` (HTTP, user-controlled)
17
+ * - `KG_EMBED_PROVIDER=voyage` (HTTPS, paid)
18
+ * - `KG_EMBED_PROVIDER=openai` (HTTPS, paid)
19
+ *
20
+ * **Failure mode** (CLAUDE.md §AI Rules #4): if the embedder fails during
21
+ * ingest, the ingester logs a warning and inserts the node + chunks WITHOUT
22
+ * vec rows. Query-time degrades gracefully to FTS5-only for affected chunks.
23
+ * The contract here is that `embed()` may throw — callers must catch.
24
+ *
25
+ * **Singleton**: model load is ~25MB and takes ~800ms cold. We load once
26
+ * per process via `getDefaultEmbedder()` and share the instance. The
27
+ * `warmup()` method preloads the model proactively at server startup.
28
+ */
29
+ /** Embedding output dimension for MiniLM. Locked — changing requires a vec table rebuild. */
30
+ export const EMBEDDING_DIM = 384;
31
+ // ----------------------------------------------------------------------------
32
+ // TransformersEmbedder — Xenova/all-MiniLM-L6-v2-quantized
33
+ // ----------------------------------------------------------------------------
34
+ const DEFAULT_MODEL = 'Xenova/all-MiniLM-L6-v2';
35
+ /**
36
+ * MiniLM via @xenova/transformers, running locally in-process. No network
37
+ * calls after first model download. The model is cached under
38
+ * `~/.cache/huggingface/` (or `XENOVA_CACHE_DIR` if set) so subsequent
39
+ * starts are fast (<500ms).
40
+ *
41
+ * Output: 384-dim Float32Array, mean-pooled across tokens, L2-normalized.
42
+ * The `pooling: 'mean'` and `normalize: true` options match the upstream
43
+ * Sentence Transformers reference inference, so embeddings produced here
44
+ * are interchangeable with embeddings produced by the Python lib.
45
+ */
46
+ export class TransformersEmbedder {
47
+ modelName;
48
+ dim = EMBEDDING_DIM;
49
+ pipelinePromise = null;
50
+ constructor(modelName = DEFAULT_MODEL) {
51
+ this.modelName = modelName;
52
+ // Allow Xenova to use local cache; only fall back to remote on first run.
53
+ // Setting allowLocalModels=true is the default but explicit here for clarity.
54
+ env.allowLocalModels = true;
55
+ env.allowRemoteModels = true;
56
+ }
57
+ async warmup() {
58
+ await this.getPipeline();
59
+ }
60
+ async embed(text) {
61
+ const extractor = await this.getPipeline();
62
+ const tensor = await extractor(text, { pooling: 'mean', normalize: true });
63
+ // Tensor.data is a Float32Array of shape [1, 384]; flatten to length 384.
64
+ const data = tensor.data;
65
+ if (data.length !== this.dim) {
66
+ throw new Error(`embedder produced ${data.length}-dim vector but ${this.dim} was expected — wrong model loaded?`);
67
+ }
68
+ // Defensive copy: tensor.data may be reused on the next call.
69
+ return new Float32Array(data);
70
+ }
71
+ async getPipeline() {
72
+ if (!this.pipelinePromise) {
73
+ logger.info({ model: this.modelName }, 'loading transformers embedder');
74
+ const t0 = Date.now();
75
+ this.pipelinePromise = pipeline('feature-extraction', this.modelName, { quantized: true });
76
+ this.pipelinePromise
77
+ .then(() => {
78
+ logger.info({ model: this.modelName, ms: Date.now() - t0 }, 'transformers embedder ready');
79
+ })
80
+ .catch((err) => {
81
+ logger.error({ err, model: this.modelName }, 'transformers embedder failed to load');
82
+ // Reset so subsequent calls can retry rather than reusing a rejected promise.
83
+ this.pipelinePromise = null;
84
+ });
85
+ }
86
+ return this.pipelinePromise;
87
+ }
88
+ }
89
+ // ----------------------------------------------------------------------------
90
+ // CountingEmbedder — test wrapper used by the per-chunk skip-unchanged test
91
+ // ----------------------------------------------------------------------------
92
+ /**
93
+ * Test-only wrapper that delegates to a real embedder and counts the number
94
+ * of `embed()` calls. Used by `ingester.test.ts` to verify that re-ingesting
95
+ * a file with one mutated paragraph triggers exactly one new embedder call —
96
+ * the load-bearing per-chunk skip-unchanged optimization (Loop 6.5 A4).
97
+ *
98
+ * Lives next to the production embedder (not under `__tests__/`) so other
99
+ * test files can import it without circular dep issues.
100
+ */
101
+ export class CountingEmbedder {
102
+ inner;
103
+ dim;
104
+ /** Number of times `embed()` has been called since construction (or `reset()`). */
105
+ calls = 0;
106
+ constructor(inner) {
107
+ this.inner = inner;
108
+ this.dim = inner.dim;
109
+ }
110
+ async warmup() {
111
+ await this.inner.warmup();
112
+ }
113
+ async embed(text) {
114
+ this.calls++;
115
+ return this.inner.embed(text);
116
+ }
117
+ reset() {
118
+ this.calls = 0;
119
+ }
120
+ }
121
+ // ----------------------------------------------------------------------------
122
+ // OllamaEmbedder — HTTP to local Ollama instance
123
+ // ----------------------------------------------------------------------------
124
+ /**
125
+ * Ollama embedder via HTTP POST to `/api/embeddings`.
126
+ * Requires `KG_OLLAMA_URL` and `KG_OLLAMA_MODEL` env vars.
127
+ */
128
+ export class OllamaEmbedder {
129
+ baseUrl;
130
+ model;
131
+ dim;
132
+ constructor(baseUrl, model, dim = EMBEDDING_DIM) {
133
+ this.baseUrl = baseUrl;
134
+ this.model = model;
135
+ this.dim = dim;
136
+ }
137
+ async warmup() {
138
+ // Ollama loads the model on first request; nothing to pre-warm.
139
+ }
140
+ async embed(text) {
141
+ const res = await fetch(`${this.baseUrl}/api/embeddings`, {
142
+ method: 'POST',
143
+ headers: { 'Content-Type': 'application/json' },
144
+ body: JSON.stringify({ model: this.model, prompt: text }),
145
+ });
146
+ if (!res.ok) {
147
+ throw new Error(`Ollama embed failed: ${res.status} ${res.statusText}`);
148
+ }
149
+ const json = (await res.json());
150
+ const vec = new Float32Array(json.embedding);
151
+ if (vec.length !== this.dim) {
152
+ throw new Error(`Ollama returned ${vec.length}-dim embedding but ${this.dim} expected`);
153
+ }
154
+ return vec;
155
+ }
156
+ }
157
+ // ----------------------------------------------------------------------------
158
+ // VoyageEmbedder — HTTPS to Voyage AI
159
+ // ----------------------------------------------------------------------------
160
+ /**
161
+ * Voyage AI embedder via HTTPS POST to `https://api.voyageai.com/v1/embeddings`.
162
+ * Requires `KG_VOYAGE_API_KEY` env var.
163
+ */
164
+ export class VoyageEmbedder {
165
+ apiKey;
166
+ model;
167
+ dim;
168
+ constructor(apiKey, model = 'voyage-code-3', dim = EMBEDDING_DIM) {
169
+ this.apiKey = apiKey;
170
+ this.model = model;
171
+ this.dim = dim;
172
+ }
173
+ async warmup() { }
174
+ async embed(text) {
175
+ const res = await fetch('https://api.voyageai.com/v1/embeddings', {
176
+ method: 'POST',
177
+ headers: {
178
+ 'Content-Type': 'application/json',
179
+ Authorization: `Bearer ${this.apiKey}`,
180
+ },
181
+ body: JSON.stringify({ model: this.model, input: text }),
182
+ });
183
+ if (!res.ok) {
184
+ throw new Error(`Voyage embed failed: ${res.status} ${res.statusText}`);
185
+ }
186
+ const json = (await res.json());
187
+ const vec = new Float32Array(json.data[0].embedding);
188
+ if (vec.length !== this.dim) {
189
+ throw new Error(`Voyage returned ${vec.length}-dim embedding but ${this.dim} expected`);
190
+ }
191
+ return vec;
192
+ }
193
+ }
194
+ // ----------------------------------------------------------------------------
195
+ // OpenAIEmbedder — HTTPS to OpenAI
196
+ // ----------------------------------------------------------------------------
197
+ /**
198
+ * OpenAI embedder via HTTPS POST to `https://api.openai.com/v1/embeddings`.
199
+ * Requires `KG_OPENAI_API_KEY` env var.
200
+ */
201
+ export class OpenAIEmbedder {
202
+ apiKey;
203
+ model;
204
+ dim;
205
+ constructor(apiKey, model = 'text-embedding-3-small', dim = EMBEDDING_DIM) {
206
+ this.apiKey = apiKey;
207
+ this.model = model;
208
+ this.dim = dim;
209
+ }
210
+ async warmup() { }
211
+ async embed(text) {
212
+ const res = await fetch('https://api.openai.com/v1/embeddings', {
213
+ method: 'POST',
214
+ headers: {
215
+ 'Content-Type': 'application/json',
216
+ Authorization: `Bearer ${this.apiKey}`,
217
+ },
218
+ body: JSON.stringify({ model: this.model, input: text }),
219
+ });
220
+ if (!res.ok) {
221
+ throw new Error(`OpenAI embed failed: ${res.status} ${res.statusText}`);
222
+ }
223
+ const json = (await res.json());
224
+ const vec = new Float32Array(json.data[0].embedding);
225
+ if (vec.length !== this.dim) {
226
+ throw new Error(`OpenAI returned ${vec.length}-dim embedding but ${this.dim} expected`);
227
+ }
228
+ return vec;
229
+ }
230
+ }
231
+ /**
232
+ * Create an embedder instance based on the provider name and environment.
233
+ * Used at server startup. Falls back to TransformersEmbedder if the
234
+ * provider is unrecognized.
235
+ */
236
+ export function createEmbedder(provider) {
237
+ const p = provider ?? process.env['KG_EMBED_PROVIDER'] ?? 'transformers';
238
+ switch (p) {
239
+ case 'ollama': {
240
+ const url = process.env['KG_OLLAMA_URL'] ?? 'http://localhost:11434';
241
+ const model = process.env['KG_OLLAMA_MODEL'] ?? 'nomic-embed-text';
242
+ return new OllamaEmbedder(url, model);
243
+ }
244
+ case 'voyage': {
245
+ const key = process.env['KG_VOYAGE_API_KEY'];
246
+ if (!key)
247
+ throw new Error('KG_VOYAGE_API_KEY is required for voyage embedder');
248
+ const model = process.env['KG_EMBED_MODEL'] ?? 'voyage-code-3';
249
+ return new VoyageEmbedder(key, model);
250
+ }
251
+ case 'openai': {
252
+ const key = process.env['KG_OPENAI_API_KEY'];
253
+ if (!key)
254
+ throw new Error('KG_OPENAI_API_KEY is required for openai embedder');
255
+ const model = process.env['KG_EMBED_MODEL'] ?? 'text-embedding-3-small';
256
+ return new OpenAIEmbedder(key, model);
257
+ }
258
+ case 'transformers':
259
+ default:
260
+ return getDefaultEmbedder();
261
+ }
262
+ }
263
+ // ----------------------------------------------------------------------------
264
+ // Default factory + singleton
265
+ // ----------------------------------------------------------------------------
266
+ let defaultEmbedder = null;
267
+ /**
268
+ * Singleton accessor. Always returns the same `TransformersEmbedder` instance
269
+ * within a process so the model loads exactly once. Tests that need a fresh
270
+ * instance can construct `new TransformersEmbedder()` directly.
271
+ */
272
+ export function getDefaultEmbedder() {
273
+ if (!defaultEmbedder) {
274
+ defaultEmbedder = new TransformersEmbedder();
275
+ }
276
+ return defaultEmbedder;
277
+ }
278
+ //# sourceMappingURL=embedder.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"embedder.js","sourceRoot":"","sources":["../../src/retrieval/embedder.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,GAAG,EAAkC,MAAM,sBAAsB,CAAC;AAErF,OAAO,EAAE,MAAM,EAAE,MAAM,4BAA4B,CAAC;AAEpD;;;;;;;;;;;;;;;;;;;;;;;;;GAyBG;AAEH,6FAA6F;AAC7F,MAAM,CAAC,MAAM,aAAa,GAAG,GAAG,CAAC;AAgBjC,+EAA+E;AAC/E,2DAA2D;AAC3D,+EAA+E;AAE/E,MAAM,aAAa,GAAG,yBAAyB,CAAC;AAEhD;;;;;;;;;;GAUG;AACH,MAAM,OAAO,oBAAoB;IAKF;IAJpB,GAAG,GAAG,aAAa,CAAC;IAErB,eAAe,GAA8C,IAAI,CAAC;IAE1E,YAA6B,YAAoB,aAAa;QAAjC,cAAS,GAAT,SAAS,CAAwB;QAC5D,0EAA0E;QAC1E,8EAA8E;QAC9E,GAAG,CAAC,gBAAgB,GAAG,IAAI,CAAC;QAC5B,GAAG,CAAC,iBAAiB,GAAG,IAAI,CAAC;IAC/B,CAAC;IAED,KAAK,CAAC,MAAM;QACV,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;IAC3B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,SAAS,GAAG,MAAM,IAAI,CAAC,WAAW,EAAE,CAAC;QAC3C,MAAM,MAAM,GAAG,MAAM,SAAS,CAAC,IAAI,EAAE,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAC;QAC3E,0EAA0E;QAC1E,MAAM,IAAI,GAAG,MAAM,CAAC,IAAoB,CAAC;QACzC,IAAI,IAAI,CAAC,MAAM,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;YAC7B,MAAM,IAAI,KAAK,CACb,qBAAqB,IAAI,CAAC,MAAM,mBAAmB,IAAI,CAAC,GAAG,qCAAqC,CACjG,CAAC;QACJ,CAAC;QACD,8DAA8D;QAC9D,OAAO,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAEO,KAAK,CAAC,WAAW;QACvB,IAAI,CAAC,IAAI,CAAC,eAAe,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,EAAE,+BAA+B,CAAC,CAAC;YACxE,MAAM,EAAE,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;YACtB,IAAI,CAAC,eAAe,GAAG,QAAQ,CAAC,oBAAoB,EAAE,IAAI,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAExF,CAAC;YACF,IAAI,CAAC,eAAe;iBACjB,IAAI,CAAC,GAAG,EAAE;gBACT,MAAM,CAAC,IAAI,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,GAAG,EAAE,EAAE,EAAE,6BAA6B,CAAC,CAAC;YAC7F,CAAC,CAAC;iBACD,KAAK,CAAC,CAAC,GAAY,EAAE,EAAE;gBACtB,MAAM,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,KAAK,EAAE,IAAI,CAAC,SAAS,EAAE,EAAE,sCAAsC,CAAC,CAAC;gBACrF,8EAA8E;gBAC9E,IAAI,CAAC,eAAe,GAAG,IAAI,CAAC;YAC9B,CAAC,CAAC,CAAC;QACP,CAAC;QACD,OAAO,IAAI,CAAC,eAAe,CAAC;IAC9B,CAAC;CACF;AAED,+EAA+E;AAC/E,4EAA4E;AAC5E,+EAA+E;AAE/E;;;;;;;;GAQG;AACH,MAAM,OAAO,gBAAgB;IAKE;IAJpB,GAAG,CAAS;IACrB,mFAAmF;IACnF,KAAK,GAAG,CAAC,CAAC;IAEV,YAA6B,KAAe;QAAf,UAAK,GAAL,KAAK,CAAU;QAC1C,IAAI,CAAC,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC;IACvB,CAAC;IAED,KAAK,CAAC,MAAM;QACV,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,EAAE,CAAC;IAC5B,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,IAAI,CAAC,KAAK,EAAE,CAAC;QACb,OAAO,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAChC,CAAC;IAED,KAAK;QACH,IAAI,CAAC,KAAK,GAAG,CAAC,CAAC;IACjB,CAAC;CACF;AAED,+EAA+E;AAC/E,iDAAiD;AACjD,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,OAAO,cAAc;IAIN;IACA;IAJV,GAAG,CAAS;IAErB,YACmB,OAAe,EACf,KAAa,EAC9B,MAAc,aAAa;QAFV,YAAO,GAAP,OAAO,CAAQ;QACf,UAAK,GAAL,KAAK,CAAQ;QAG9B,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,MAAM;QACV,gEAAgE;IAClE,CAAC;IAED,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,GAAG,IAAI,CAAC,OAAO,iBAAiB,EAAE;YACxD,MAAM,EAAE,MAAM;YACd,OAAO,EAAE,EAAE,cAAc,EAAE,kBAAkB,EAAE;YAC/C,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC;SAC1D,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA4B,CAAC;QAC3D,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;QAC7C,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACb,mBAAmB,GAAG,CAAC,MAAM,sBAAsB,IAAI,CAAC,GAAG,WAAW,CACvE,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAED,+EAA+E;AAC/E,sCAAsC;AACtC,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,OAAO,cAAc;IAIN;IACA;IAJV,GAAG,CAAS;IAErB,YACmB,MAAc,EACd,QAAgB,eAAe,EAChD,MAAc,aAAa;QAFV,WAAM,GAAN,MAAM,CAAQ;QACd,UAAK,GAAL,KAAK,CAA0B;QAGhD,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,MAAM,KAAmB,CAAC;IAEhC,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,wCAAwC,EAAE;YAChE,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;SACzD,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA6C,CAAC;QAC5E,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACrD,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACb,mBAAmB,GAAG,CAAC,MAAM,sBAAsB,IAAI,CAAC,GAAG,WAAW,CACvE,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAED,+EAA+E;AAC/E,mCAAmC;AACnC,+EAA+E;AAE/E;;;GAGG;AACH,MAAM,OAAO,cAAc;IAIN;IACA;IAJV,GAAG,CAAS;IAErB,YACmB,MAAc,EACd,QAAgB,wBAAwB,EACzD,MAAc,aAAa;QAFV,WAAM,GAAN,MAAM,CAAQ;QACd,UAAK,GAAL,KAAK,CAAmC;QAGzD,IAAI,CAAC,GAAG,GAAG,GAAG,CAAC;IACjB,CAAC;IAED,KAAK,CAAC,MAAM,KAAmB,CAAC;IAEhC,KAAK,CAAC,KAAK,CAAC,IAAY;QACtB,MAAM,GAAG,GAAG,MAAM,KAAK,CAAC,sCAAsC,EAAE;YAC9D,MAAM,EAAE,MAAM;YACd,OAAO,EAAE;gBACP,cAAc,EAAE,kBAAkB;gBAClC,aAAa,EAAE,UAAU,IAAI,CAAC,MAAM,EAAE;aACvC;YACD,IAAI,EAAE,IAAI,CAAC,SAAS,CAAC,EAAE,KAAK,EAAE,IAAI,CAAC,KAAK,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;SACzD,CAAC,CAAC;QACH,IAAI,CAAC,GAAG,CAAC,EAAE,EAAE,CAAC;YACZ,MAAM,IAAI,KAAK,CAAC,wBAAwB,GAAG,CAAC,MAAM,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC,CAAC;QAC1E,CAAC;QACD,MAAM,IAAI,GAAG,CAAC,MAAM,GAAG,CAAC,IAAI,EAAE,CAA6C,CAAC;QAC5E,MAAM,GAAG,GAAG,IAAI,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACrD,IAAI,GAAG,CAAC,MAAM,KAAK,IAAI,CAAC,GAAG,EAAE,CAAC;YAC5B,MAAM,IAAI,KAAK,CACb,mBAAmB,GAAG,CAAC,MAAM,sBAAsB,IAAI,CAAC,GAAG,WAAW,CACvE,CAAC;QACJ,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC;CACF;AAQD;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,QAAwB;IACrD,MAAM,CAAC,GAAG,QAAQ,IAAK,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAA+B,IAAI,cAAc,CAAC;IAExG,QAAQ,CAAC,EAAE,CAAC;QACV,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,eAAe,CAAC,IAAI,wBAAwB,CAAC;YACrE,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC,IAAI,kBAAkB,CAAC;YACnE,OAAO,IAAI,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACxC,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;YAC7C,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;YAC/E,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,eAAe,CAAC;YAC/D,OAAO,IAAI,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACxC,CAAC;QACD,KAAK,QAAQ,CAAC,CAAC,CAAC;YACd,MAAM,GAAG,GAAG,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC,CAAC;YAC7C,IAAI,CAAC,GAAG;gBAAE,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;YAC/E,MAAM,KAAK,GAAG,OAAO,CAAC,GAAG,CAAC,gBAAgB,CAAC,IAAI,wBAAwB,CAAC;YACxE,OAAO,IAAI,cAAc,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;QACxC,CAAC;QACD,KAAK,cAAc,CAAC;QACpB;YACE,OAAO,kBAAkB,EAAE,CAAC;IAChC,CAAC;AACH,CAAC;AAED,+EAA+E;AAC/E,8BAA8B;AAC9B,+EAA+E;AAE/E,IAAI,eAAe,GAAgC,IAAI,CAAC;AAExD;;;;GAIG;AACH,MAAM,UAAU,kBAAkB;IAChC,IAAI,CAAC,eAAe,EAAE,CAAC;QACrB,eAAe,GAAG,IAAI,oBAAoB,EAAE,CAAC;IAC/C,CAAC;IACD,OAAO,eAAe,CAAC;AACzB,CAAC"}