@askdkc/kiokuko 0.2.6 → 0.2.7

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (167) hide show
  1. package/README.ja.md +31 -0
  2. package/README.ko.md +31 -0
  3. package/README.md +32 -0
  4. package/README.zh-CN.md +29 -0
  5. package/dist/akinator/agent-task.d.ts +3 -0
  6. package/dist/akinator/agent-task.d.ts.map +1 -1
  7. package/dist/akinator/agent-task.js +38 -4
  8. package/dist/akinator/agent-task.js.map +1 -1
  9. package/dist/cli.d.ts +4 -0
  10. package/dist/cli.d.ts.map +1 -1
  11. package/dist/cli.js +89 -8
  12. package/dist/cli.js.map +1 -1
  13. package/dist/commands/doctor.d.ts +4 -0
  14. package/dist/commands/doctor.d.ts.map +1 -1
  15. package/dist/commands/doctor.js +43 -1
  16. package/dist/commands/doctor.js.map +1 -1
  17. package/dist/commands/embeddings.d.ts +19 -0
  18. package/dist/commands/embeddings.d.ts.map +1 -0
  19. package/dist/commands/embeddings.js +159 -0
  20. package/dist/commands/embeddings.js.map +1 -0
  21. package/dist/commands/import.d.ts.map +1 -1
  22. package/dist/commands/import.js +7 -0
  23. package/dist/commands/import.js.map +1 -1
  24. package/dist/commands/use.d.ts.map +1 -1
  25. package/dist/commands/use.js +22 -8
  26. package/dist/commands/use.js.map +1 -1
  27. package/dist/context/broker.d.ts +3 -1
  28. package/dist/context/broker.d.ts.map +1 -1
  29. package/dist/context/broker.js +11 -7
  30. package/dist/context/broker.js.map +1 -1
  31. package/dist/context/delivery.d.ts.map +1 -1
  32. package/dist/context/delivery.js +5 -3
  33. package/dist/context/delivery.js.map +1 -1
  34. package/dist/context/ranking.d.ts +1 -1
  35. package/dist/context/ranking.d.ts.map +1 -1
  36. package/dist/context/ranking.js +1 -0
  37. package/dist/context/ranking.js.map +1 -1
  38. package/dist/context/scoped-broker.d.ts +4 -3
  39. package/dist/context/scoped-broker.d.ts.map +1 -1
  40. package/dist/context/scoped-broker.js +45 -7
  41. package/dist/context/scoped-broker.js.map +1 -1
  42. package/dist/context/selection-state.d.ts +12 -0
  43. package/dist/context/selection-state.d.ts.map +1 -1
  44. package/dist/context/selection-state.js +117 -7
  45. package/dist/context/selection-state.js.map +1 -1
  46. package/dist/db/capabilities.d.ts +8 -0
  47. package/dist/db/capabilities.d.ts.map +1 -1
  48. package/dist/db/capabilities.js +14 -0
  49. package/dist/db/capabilities.js.map +1 -1
  50. package/dist/db/connection.d.ts +7 -0
  51. package/dist/db/connection.d.ts.map +1 -1
  52. package/dist/db/connection.js +56 -0
  53. package/dist/db/connection.js.map +1 -1
  54. package/dist/db/migrate.d.ts.map +1 -1
  55. package/dist/db/migrate.js +3 -0
  56. package/dist/db/migrate.js.map +1 -1
  57. package/dist/embedding/backend.d.ts +27 -0
  58. package/dist/embedding/backend.d.ts.map +1 -0
  59. package/dist/embedding/backend.js +51 -0
  60. package/dist/embedding/backend.js.map +1 -0
  61. package/dist/embedding/config.d.ts +7 -0
  62. package/dist/embedding/config.d.ts.map +1 -0
  63. package/dist/embedding/config.js +125 -0
  64. package/dist/embedding/config.js.map +1 -0
  65. package/dist/embedding/diagnostics.d.ts +34 -0
  66. package/dist/embedding/diagnostics.d.ts.map +1 -0
  67. package/dist/embedding/diagnostics.js +248 -0
  68. package/dist/embedding/diagnostics.js.map +1 -0
  69. package/dist/embedding/document.d.ts +15 -0
  70. package/dist/embedding/document.d.ts.map +1 -0
  71. package/dist/embedding/document.js +102 -0
  72. package/dist/embedding/document.js.map +1 -0
  73. package/dist/embedding/javascript-backend.d.ts +11 -0
  74. package/dist/embedding/javascript-backend.d.ts.map +1 -0
  75. package/dist/embedding/javascript-backend.js +164 -0
  76. package/dist/embedding/javascript-backend.js.map +1 -0
  77. package/dist/embedding/jobs.d.ts +67 -0
  78. package/dist/embedding/jobs.d.ts.map +1 -0
  79. package/dist/embedding/jobs.js +259 -0
  80. package/dist/embedding/jobs.js.map +1 -0
  81. package/dist/embedding/openai-compatible-provider.d.ts +24 -0
  82. package/dist/embedding/openai-compatible-provider.d.ts.map +1 -0
  83. package/dist/embedding/openai-compatible-provider.js +330 -0
  84. package/dist/embedding/openai-compatible-provider.js.map +1 -0
  85. package/dist/embedding/profile.d.ts +6 -0
  86. package/dist/embedding/profile.d.ts.map +1 -0
  87. package/dist/embedding/profile.js +31 -0
  88. package/dist/embedding/profile.js.map +1 -0
  89. package/dist/embedding/provider.d.ts +13 -0
  90. package/dist/embedding/provider.d.ts.map +1 -0
  91. package/dist/embedding/provider.js +17 -0
  92. package/dist/embedding/provider.js.map +1 -0
  93. package/dist/embedding/query-cache.d.ts +39 -0
  94. package/dist/embedding/query-cache.d.ts.map +1 -0
  95. package/dist/embedding/query-cache.js +160 -0
  96. package/dist/embedding/query-cache.js.map +1 -0
  97. package/dist/embedding/runtime.d.ts +13 -0
  98. package/dist/embedding/runtime.d.ts.map +1 -0
  99. package/dist/embedding/runtime.js +373 -0
  100. package/dist/embedding/runtime.js.map +1 -0
  101. package/dist/embedding/sqlite-vec-backend.d.ts +9 -0
  102. package/dist/embedding/sqlite-vec-backend.d.ts.map +1 -0
  103. package/dist/embedding/sqlite-vec-backend.js +106 -0
  104. package/dist/embedding/sqlite-vec-backend.js.map +1 -0
  105. package/dist/embedding/sqlite-vec-loader.d.ts +17 -0
  106. package/dist/embedding/sqlite-vec-loader.d.ts.map +1 -0
  107. package/dist/embedding/sqlite-vec-loader.js +51 -0
  108. package/dist/embedding/sqlite-vec-loader.js.map +1 -0
  109. package/dist/embedding/store.d.ts +64 -0
  110. package/dist/embedding/store.d.ts.map +1 -0
  111. package/dist/embedding/store.js +377 -0
  112. package/dist/embedding/store.js.map +1 -0
  113. package/dist/embedding/types.d.ts +97 -0
  114. package/dist/embedding/types.d.ts.map +1 -0
  115. package/dist/embedding/types.js +2 -0
  116. package/dist/embedding/types.js.map +1 -0
  117. package/dist/embedding/vector.d.ts +9 -0
  118. package/dist/embedding/vector.d.ts.map +1 -0
  119. package/dist/embedding/vector.js +90 -0
  120. package/dist/embedding/vector.js.map +1 -0
  121. package/dist/embedding/worker.d.ts +23 -0
  122. package/dist/embedding/worker.d.ts.map +1 -0
  123. package/dist/embedding/worker.js +84 -0
  124. package/dist/embedding/worker.js.map +1 -0
  125. package/dist/enno-oduno/schemas.d.ts +2 -2
  126. package/dist/ledger/checkpoint-contract.d.ts +2 -2
  127. package/dist/mcp/bounded-stdio-transport.d.ts +12 -2
  128. package/dist/mcp/bounded-stdio-transport.d.ts.map +1 -1
  129. package/dist/mcp/bounded-stdio-transport.js +103 -4
  130. package/dist/mcp/bounded-stdio-transport.js.map +1 -1
  131. package/dist/mcp/runtime-owner.d.ts +27 -0
  132. package/dist/mcp/runtime-owner.d.ts.map +1 -0
  133. package/dist/mcp/runtime-owner.js +130 -0
  134. package/dist/mcp/runtime-owner.js.map +1 -0
  135. package/dist/mcp/server.d.ts +9 -1
  136. package/dist/mcp/server.d.ts.map +1 -1
  137. package/dist/mcp/server.js +38 -5
  138. package/dist/mcp/server.js.map +1 -1
  139. package/dist/memory/checkpoint-contract.d.ts +3 -3
  140. package/dist/memory/entries.d.ts.map +1 -1
  141. package/dist/memory/entries.js +13 -0
  142. package/dist/memory/entries.js.map +1 -1
  143. package/dist/memory/federated-retrieval.d.ts +3 -2
  144. package/dist/memory/federated-retrieval.d.ts.map +1 -1
  145. package/dist/memory/federated-retrieval.js +112 -31
  146. package/dist/memory/federated-retrieval.js.map +1 -1
  147. package/dist/memory/hybrid-retrieval.d.ts +9 -2
  148. package/dist/memory/hybrid-retrieval.d.ts.map +1 -1
  149. package/dist/memory/hybrid-retrieval.js +60 -13
  150. package/dist/memory/hybrid-retrieval.js.map +1 -1
  151. package/dist/memory/retrieval.d.ts +4 -3
  152. package/dist/memory/retrieval.d.ts.map +1 -1
  153. package/dist/memory/retrieval.js +6 -6
  154. package/dist/memory/retrieval.js.map +1 -1
  155. package/dist/memory/scoped-memory.d.ts +2 -1
  156. package/dist/memory/scoped-memory.d.ts.map +1 -1
  157. package/dist/memory/scoped-memory.js +2 -2
  158. package/dist/memory/scoped-memory.js.map +1 -1
  159. package/dist/server/agent-application.js +1 -1
  160. package/dist/server/agent-application.js.map +1 -1
  161. package/dist/server/http.d.ts +17 -1
  162. package/dist/server/http.d.ts.map +1 -1
  163. package/dist/server/http.js +76 -13
  164. package/dist/server/http.js.map +1 -1
  165. package/migrations/021_semantic_embeddings.sql +122 -0
  166. package/migrations/down/021_semantic_embeddings.sql +17 -0
  167. package/package.json +7 -1
@@ -0,0 +1,125 @@
1
+ import { KiokukoError } from '../errors.js';
2
+ const DEFAULT_TIMEOUT_MS = 30_000;
3
+ const DEFAULT_BATCH_SIZE = 16;
4
+ const MAX_MODEL_LENGTH = 256;
5
+ const MAX_API_KEY_LENGTH = 4_096;
6
+ const LOOPBACK_HOSTS = new Set(['localhost', '127.0.0.1', '::1']);
7
+ function invalid(message) {
8
+ throw new KiokukoError('VALIDATION_ERROR', message);
9
+ }
10
+ function enumValue(value, fallback, allowed, field) {
11
+ const selected = value === undefined ? fallback : value;
12
+ if (!allowed.includes(selected))
13
+ invalid(`${field} has an unsupported value`);
14
+ return selected;
15
+ }
16
+ function boundedText(value, field, maxLength) {
17
+ const normalized = value.normalize('NFKC').trim();
18
+ if (normalized.length === 0 || normalized.length > maxLength || /[\u0000-\u001f\u007f]/u.test(normalized)) {
19
+ invalid(`${field} is invalid`);
20
+ }
21
+ return normalized;
22
+ }
23
+ function optionalText(value, field, maxLength) {
24
+ return value === undefined || value.length === 0 ? undefined : boundedText(value, field, maxLength);
25
+ }
26
+ function integerValue(value, fallback, minimum, maximum, field) {
27
+ if (value === undefined)
28
+ return fallback;
29
+ if (!/^\d+$/u.test(value))
30
+ invalid(`${field} must be an integer in the supported range`);
31
+ const parsed = Number(value);
32
+ if (!Number.isSafeInteger(parsed) || parsed < minimum || parsed > maximum) {
33
+ invalid(`${field} must be an integer in the supported range`);
34
+ }
35
+ return parsed;
36
+ }
37
+ function distanceCeilingValue(value) {
38
+ if (value === undefined)
39
+ return undefined;
40
+ if (!/^(?:\d+\.?\d*|\.\d+)$/u.test(value))
41
+ invalid('KIOKUKO_EMBEDDING_DISTANCE_CEILING is invalid');
42
+ const parsed = Number(value);
43
+ if (!Number.isFinite(parsed) || parsed <= 0 || parsed >= 2) {
44
+ invalid('KIOKUKO_EMBEDDING_DISTANCE_CEILING is invalid');
45
+ }
46
+ return parsed;
47
+ }
48
+ function booleanValue(value, fallback, field) {
49
+ if (value === undefined)
50
+ return fallback;
51
+ if (value !== 'true' && value !== 'false')
52
+ invalid(`${field} must be true or false`);
53
+ return value === 'true';
54
+ }
55
+ /** Normalize and validate an embedding endpoint without retaining credentials. */
56
+ export function normalizeEmbeddingBaseUrl(value, allowRemote) {
57
+ const input = boundedText(value, 'KIOKUKO_EMBEDDING_BASE_URL', 2_048);
58
+ let url;
59
+ try {
60
+ url = new URL(input);
61
+ }
62
+ catch {
63
+ invalid('KIOKUKO_EMBEDDING_BASE_URL must be an absolute URL');
64
+ }
65
+ if (url.username !== '' || url.password !== '' || url.search !== '' || url.hash !== '') {
66
+ invalid('KIOKUKO_EMBEDDING_BASE_URL must not contain credentials, query, or fragment data');
67
+ }
68
+ const hostname = url.hostname.replace(/^\[|\]$/gu, '').toLowerCase();
69
+ if (url.protocol === 'http:') {
70
+ if (!LOOPBACK_HOSTS.has(hostname))
71
+ invalid('HTTP embedding endpoints must use a loopback host');
72
+ }
73
+ else if (url.protocol === 'https:') {
74
+ if (!allowRemote)
75
+ invalid('HTTPS embedding endpoints require explicit remote opt-in');
76
+ }
77
+ else {
78
+ invalid('KIOKUKO_EMBEDDING_BASE_URL must use HTTP or HTTPS');
79
+ }
80
+ return url.toString();
81
+ }
82
+ export function isEnabledEmbeddingConfig(config) {
83
+ return config.mode !== 'off'
84
+ && config.baseUrl !== undefined
85
+ && config.model !== undefined
86
+ && config.dimensions !== undefined
87
+ && config.distanceCeiling !== undefined;
88
+ }
89
+ export function requireEnabledEmbeddingConfig(config) {
90
+ if (!isEnabledEmbeddingConfig(config))
91
+ invalid('Embedding configuration is disabled or incomplete');
92
+ return config;
93
+ }
94
+ export function parseEmbeddingConfig(environment = process.env) {
95
+ const mode = enumValue(environment.KIOKUKO_EMBEDDINGS, 'off', ['off', 'optional', 'required'], 'KIOKUKO_EMBEDDINGS');
96
+ const provider = enumValue(environment.KIOKUKO_EMBEDDING_PROVIDER, 'openai-compatible', ['openai-compatible'], 'KIOKUKO_EMBEDDING_PROVIDER');
97
+ const allowRemote = booleanValue(environment.KIOKUKO_EMBEDDING_ALLOW_REMOTE, false, 'KIOKUKO_EMBEDDING_ALLOW_REMOTE');
98
+ const vectorBackend = enumValue(environment.KIOKUKO_VECTOR_BACKEND, 'auto', ['auto', 'javascript', 'sqlite-vec'], 'KIOKUKO_VECTOR_BACKEND');
99
+ const baseUrl = environment.KIOKUKO_EMBEDDING_BASE_URL === undefined
100
+ ? undefined
101
+ : normalizeEmbeddingBaseUrl(environment.KIOKUKO_EMBEDDING_BASE_URL, allowRemote);
102
+ const model = optionalText(environment.KIOKUKO_EMBEDDING_MODEL, 'KIOKUKO_EMBEDDING_MODEL', MAX_MODEL_LENGTH);
103
+ const dimensions = integerValue(environment.KIOKUKO_EMBEDDING_DIMENSIONS, 0, 2, 8_192, 'KIOKUKO_EMBEDDING_DIMENSIONS');
104
+ const configuredDistanceCeiling = distanceCeilingValue(environment.KIOKUKO_EMBEDDING_DISTANCE_CEILING);
105
+ const timeoutMs = integerValue(environment.KIOKUKO_EMBEDDING_TIMEOUT_MS, DEFAULT_TIMEOUT_MS, 100, 120_000, 'KIOKUKO_EMBEDDING_TIMEOUT_MS');
106
+ const batchSize = integerValue(environment.KIOKUKO_EMBEDDING_BATCH_SIZE, DEFAULT_BATCH_SIZE, 1, 64, 'KIOKUKO_EMBEDDING_BATCH_SIZE');
107
+ const apiKey = optionalText(environment.KIOKUKO_EMBEDDING_API_KEY, 'KIOKUKO_EMBEDDING_API_KEY', MAX_API_KEY_LENGTH);
108
+ if (mode !== 'off' && (baseUrl === undefined || model === undefined || dimensions === 0 || configuredDistanceCeiling === undefined)) {
109
+ invalid('Enabled embeddings require base URL, model, dimensions, and distance ceiling');
110
+ }
111
+ return {
112
+ mode,
113
+ provider,
114
+ ...(baseUrl === undefined ? {} : { baseUrl }),
115
+ ...(model === undefined ? {} : { model }),
116
+ ...(dimensions === 0 ? {} : { dimensions }),
117
+ ...(configuredDistanceCeiling === undefined ? {} : { distanceCeiling: configuredDistanceCeiling }),
118
+ ...(apiKey === undefined ? {} : { apiKey }),
119
+ allowRemote,
120
+ vectorBackend,
121
+ timeoutMs,
122
+ batchSize,
123
+ };
124
+ }
125
+ //# sourceMappingURL=config.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"config.js","sourceRoot":"","sources":["../../src/embedding/config.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAG5C,MAAM,kBAAkB,GAAG,MAAM,CAAC;AAClC,MAAM,kBAAkB,GAAG,EAAE,CAAC;AAC9B,MAAM,gBAAgB,GAAG,GAAG,CAAC;AAC7B,MAAM,kBAAkB,GAAG,KAAK,CAAC;AACjC,MAAM,cAAc,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,WAAW,EAAE,KAAK,CAAC,CAAC,CAAC;AAElE,SAAS,OAAO,CAAC,OAAe;IAC9B,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,SAAS,CAAmB,KAAyB,EAAE,QAAW,EAAE,OAAqB,EAAE,KAAa;IAC/G,MAAM,QAAQ,GAAM,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAU,CAAC;IAChE,IAAI,CAAC,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,2BAA2B,CAAC,CAAC;IAC9E,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,WAAW,CAAC,KAAa,EAAE,KAAa,EAAE,SAAiB;IAClE,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC;IAClD,IAAI,UAAU,CAAC,MAAM,KAAK,CAAC,IAAI,UAAU,CAAC,MAAM,GAAG,SAAS,IAAI,wBAAwB,CAAC,IAAI,CAAC,UAAU,CAAC,EAAE,CAAC;QAC1G,OAAO,CAAC,GAAG,KAAK,aAAa,CAAC,CAAC;IACjC,CAAC;IACD,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,YAAY,CAAC,KAAyB,EAAE,KAAa,EAAE,SAAiB;IAC/E,OAAO,KAAK,KAAK,SAAS,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,EAAE,SAAS,CAAC,CAAC;AACtG,CAAC;AAED,SAAS,YAAY,CAAC,KAAyB,EAAE,QAAgB,EAAE,OAAe,EAAE,OAAe,EAAE,KAAa;IAChH,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACzC,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,4CAA4C,CAAC,CAAC;IACzF,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,MAAM,CAAC,IAAI,MAAM,GAAG,OAAO,IAAI,MAAM,GAAG,OAAO,EAAE,CAAC;QAC1E,OAAO,CAAC,GAAG,KAAK,4CAA4C,CAAC,CAAC;IAChE,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAyB;IACrD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,IAAI,CAAC,wBAAwB,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,CAAC,+CAA+C,CAAC,CAAC;IACpG,MAAM,MAAM,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC;IAC7B,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,MAAM,IAAI,CAAC,IAAI,MAAM,IAAI,CAAC,EAAE,CAAC;QAC3D,OAAO,CAAC,+CAA+C,CAAC,CAAC;IAC3D,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,YAAY,CAAC,KAAyB,EAAE,QAAiB,EAAE,KAAa;IAC/E,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,QAAQ,CAAC;IACzC,IAAI,KAAK,KAAK,MAAM,IAAI,KAAK,KAAK,OAAO;QAAE,OAAO,CAAC,GAAG,KAAK,wBAAwB,CAAC,CAAC;IACrF,OAAO,KAAK,KAAK,MAAM,CAAC;AAC1B,CAAC;AAED,kFAAkF;AAClF,MAAM,UAAU,yBAAyB,CAAC,KAAa,EAAE,WAAoB;IAC3E,MAAM,KAAK,GAAG,WAAW,CAAC,KAAK,EAAE,4BAA4B,EAAE,KAAK,CAAC,CAAC;IACtE,IAAI,GAAQ,CAAC;IACb,IAAI,CAAC;QACH,GAAG,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IACvB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,oDAAoD,CAAC,CAAC;IAChE,CAAC;IACD,IAAI,GAAG,CAAC,QAAQ,KAAK,EAAE,IAAI,GAAG,CAAC,QAAQ,KAAK,EAAE,IAAI,GAAG,CAAC,MAAM,KAAK,EAAE,IAAI,GAAG,CAAC,IAAI,KAAK,EAAE,EAAE,CAAC;QACvF,OAAO,CAAC,kFAAkF,CAAC,CAAC;IAC9F,CAAC;IACD,MAAM,QAAQ,GAAG,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IACrE,IAAI,GAAG,CAAC,QAAQ,KAAK,OAAO,EAAE,CAAC;QAC7B,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,QAAQ,CAAC;YAAE,OAAO,CAAC,mDAAmD,CAAC,CAAC;IAClG,CAAC;SAAM,IAAI,GAAG,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QACrC,IAAI,CAAC,WAAW;YAAE,OAAO,CAAC,0DAA0D,CAAC,CAAC;IACxF,CAAC;SAAM,CAAC;QACN,OAAO,CAAC,mDAAmD,CAAC,CAAC;IAC/D,CAAC;IACD,OAAO,GAAG,CAAC,QAAQ,EAAE,CAAC;AACxB,CAAC;AAED,MAAM,UAAU,wBAAwB,CAAC,MAAuB;IAC9D,OAAO,MAAM,CAAC,IAAI,KAAK,KAAK;WACvB,MAAM,CAAC,OAAO,KAAK,SAAS;WAC5B,MAAM,CAAC,KAAK,KAAK,SAAS;WAC1B,MAAM,CAAC,UAAU,KAAK,SAAS;WAC/B,MAAM,CAAC,eAAe,KAAK,SAAS,CAAC;AAC5C,CAAC;AAED,MAAM,UAAU,6BAA6B,CAAC,MAAuB;IACnE,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,mDAAmD,CAAC,CAAC;IACpG,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,MAAM,UAAU,oBAAoB,CAAC,cAAiC,OAAO,CAAC,GAAG;IAC/E,MAAM,IAAI,GAAG,SAAS,CAAgB,WAAW,CAAC,kBAAkB,EAAE,KAAK,EAAE,CAAC,KAAK,EAAE,UAAU,EAAE,UAAU,CAAC,EAAE,oBAAoB,CAAC,CAAC;IACpI,MAAM,QAAQ,GAAG,SAAS,CAAwB,WAAW,CAAC,0BAA0B,EAAE,mBAAmB,EAAE,CAAC,mBAAmB,CAAC,EAAE,4BAA4B,CAAC,CAAC;IACpK,MAAM,WAAW,GAAG,YAAY,CAAC,WAAW,CAAC,8BAA8B,EAAE,KAAK,EAAE,gCAAgC,CAAC,CAAC;IACtH,MAAM,aAAa,GAAG,SAAS,CAA0B,WAAW,CAAC,sBAAsB,EAAE,MAAM,EAAE,CAAC,MAAM,EAAE,YAAY,EAAE,YAAY,CAAC,EAAE,wBAAwB,CAAC,CAAC;IACrK,MAAM,OAAO,GAAG,WAAW,CAAC,0BAA0B,KAAK,SAAS;QAClE,CAAC,CAAC,SAAS;QACX,CAAC,CAAC,yBAAyB,CAAC,WAAW,CAAC,0BAA0B,EAAE,WAAW,CAAC,CAAC;IACnF,MAAM,KAAK,GAAG,YAAY,CAAC,WAAW,CAAC,uBAAuB,EAAE,yBAAyB,EAAE,gBAAgB,CAAC,CAAC;IAC7G,MAAM,UAAU,GAAG,YAAY,CAAC,WAAW,CAAC,4BAA4B,EAAE,CAAC,EAAE,CAAC,EAAE,KAAK,EAAE,8BAA8B,CAAC,CAAC;IACvH,MAAM,yBAAyB,GAAG,oBAAoB,CAAC,WAAW,CAAC,kCAAkC,CAAC,CAAC;IACvG,MAAM,SAAS,GAAG,YAAY,CAAC,WAAW,CAAC,4BAA4B,EAAE,kBAAkB,EAAE,GAAG,EAAE,OAAO,EAAE,8BAA8B,CAAC,CAAC;IAC3I,MAAM,SAAS,GAAG,YAAY,CAAC,WAAW,CAAC,4BAA4B,EAAE,kBAAkB,EAAE,CAAC,EAAE,EAAE,EAAE,8BAA8B,CAAC,CAAC;IACpI,MAAM,MAAM,GAAG,YAAY,CAAC,WAAW,CAAC,yBAAyB,EAAE,2BAA2B,EAAE,kBAAkB,CAAC,CAAC;IAEpH,IAAI,IAAI,KAAK,KAAK,IAAI,CAAC,OAAO,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS,IAAI,UAAU,KAAK,CAAC,IAAI,yBAAyB,KAAK,SAAS,CAAC,EAAE,CAAC;QACpI,OAAO,CAAC,8EAA8E,CAAC,CAAC;IAC1F,CAAC;IAED,OAAO;QACL,IAAI;QACJ,QAAQ;QACR,GAAG,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC;QAC7C,GAAG,CAAC,KAAK,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,CAAC;QACzC,GAAG,CAAC,UAAU,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;QAC3C,GAAG,CAAC,yBAAyB,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,eAAe,EAAE,yBAAyB,EAAE,CAAC;QAClG,GAAG,CAAC,MAAM,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,CAAC;QAC3C,WAAW;QACX,aAAa;QACb,SAAS;QACT,SAAS;KACV,CAAC;AACJ,CAAC"}
@@ -0,0 +1,34 @@
1
+ import type { SqliteDatabase } from '../db/adapter.js';
2
+ import type { EmbeddingConfig, EmbeddingMode, VectorSearchBackend } from './types.js';
3
+ export interface EmbeddingStatus {
4
+ readonly mode: EmbeddingMode;
5
+ readonly activeProfileId: string | null;
6
+ readonly providerKind: string | null;
7
+ readonly model: string | null;
8
+ readonly dimensions: number | null;
9
+ readonly distanceMetric: string | null;
10
+ readonly distanceCeiling: number | null;
11
+ readonly backend: string | null;
12
+ readonly backendVersion: string | null;
13
+ readonly eligibleEntries: number;
14
+ readonly readyVectors: number;
15
+ readonly staleVectors: number;
16
+ readonly missingVectors: number;
17
+ readonly pendingJobs: number;
18
+ readonly leasedJobs: number;
19
+ readonly failedJobs: number;
20
+ readonly blockedJobs: number;
21
+ readonly coverageRatio: number;
22
+ readonly queryCacheRows: number;
23
+ }
24
+ export interface EmbeddingHealth {
25
+ readonly status: EmbeddingStatus;
26
+ readonly check: {
27
+ readonly ok: boolean;
28
+ readonly count: number;
29
+ readonly detail: string;
30
+ };
31
+ }
32
+ export declare function readEmbeddingStatus(database: SqliteDatabase, config?: EmbeddingConfig, backend?: VectorSearchBackend): EmbeddingStatus;
33
+ export declare function inspectEmbeddingHealth(database: SqliteDatabase, environment?: NodeJS.ProcessEnv, backend?: VectorSearchBackend): EmbeddingHealth;
34
+ //# sourceMappingURL=diagnostics.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"diagnostics.d.ts","sourceRoot":"","sources":["../../src/embedding/diagnostics.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAa,MAAM,kBAAkB,CAAC;AAMlE,OAAO,KAAK,EAAE,eAAe,EAAE,aAAa,EAAoB,mBAAmB,EAAE,MAAM,YAAY,CAAC;AAgBxG,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,IAAI,EAAE,aAAa,CAAC;IAC7B,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IACrC,QAAQ,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IAC9B,QAAQ,CAAC,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;IACnC,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IACxC,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,cAAc,EAAE,MAAM,GAAG,IAAI,CAAC;IACvC,QAAQ,CAAC,eAAe,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,YAAY,EAAE,MAAM,CAAC;IAC9B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;IAChC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,UAAU,EAAE,MAAM,CAAC;IAC5B,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAC;IAC7B,QAAQ,CAAC,aAAa,EAAE,MAAM,CAAC;IAC/B,QAAQ,CAAC,cAAc,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,eAAe;IAC9B,QAAQ,CAAC,MAAM,EAAE,eAAe,CAAC;IACjC,QAAQ,CAAC,KAAK,EAAE;QACd,QAAQ,CAAC,EAAE,EAAE,OAAO,CAAC;QACrB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;QACvB,QAAQ,CAAC,MAAM,EAAE,MAAM,CAAC;KACzB,CAAC;CACH;AA+GD,wBAAgB,mBAAmB,CACjC,QAAQ,EAAE,cAAc,EACxB,MAAM,GAAE,eAAwC,EAChD,OAAO,CAAC,EAAE,mBAAmB,GAC5B,eAAe,CAOjB;AAyFD,wBAAgB,sBAAsB,CACpC,QAAQ,EAAE,cAAc,EACxB,WAAW,GAAE,MAAM,CAAC,UAAwB,EAC5C,OAAO,CAAC,EAAE,mBAAmB,GAC5B,eAAe,CA0CjB"}
@@ -0,0 +1,248 @@
1
+ import { KiokukoError } from '../errors.js';
2
+ import { parseEmbeddingConfig, isEnabledEmbeddingConfig } from './config.js';
3
+ import { embeddingProfileId, createEmbeddingProfile } from './profile.js';
4
+ import { readActiveEmbeddingProfile, readEmbeddingRuntimeState } from './store.js';
5
+ import { decodeVector, hashVectorBytes } from './vector.js';
6
+ const EMBEDDING_TABLES = [
7
+ 'embedding_profiles',
8
+ 'embedding_runtime',
9
+ 'entry_embeddings',
10
+ 'embedding_jobs',
11
+ 'query_embeddings',
12
+ ];
13
+ const EMBEDDING_INDEXES = [
14
+ 'idx_entry_embeddings_profile_revision',
15
+ 'idx_embedding_jobs_claim',
16
+ 'idx_query_embeddings_lru',
17
+ ];
18
+ function countRows(database, sql, ...parameters) {
19
+ return Number(database.prepare(sql).get(...parameters)?.count ?? 0);
20
+ }
21
+ function objectNames(database, kind) {
22
+ return new Set(database.prepare(`
23
+ SELECT name FROM sqlite_schema WHERE type = ? AND name IS NOT NULL
24
+ `).all(kind).map((row) => row.name));
25
+ }
26
+ function configuredBackend(config, backend) {
27
+ if (config.mode === 'off')
28
+ return { id: null, version: null };
29
+ if (backend !== undefined)
30
+ return { id: backend.id, version: backend.id === 'sqlite-vec' ? 'v0.1.9' : null };
31
+ if (config.vectorBackend === 'sqlite-vec')
32
+ return { id: 'sqlite-vec', version: 'v0.1.9' };
33
+ return { id: 'javascript', version: null };
34
+ }
35
+ function emptyStatus(config, backend) {
36
+ const selectedBackend = configuredBackend(config, backend);
37
+ return {
38
+ mode: config.mode,
39
+ activeProfileId: null,
40
+ providerKind: config.mode === 'off' ? null : config.provider,
41
+ model: config.model ?? null,
42
+ dimensions: config.dimensions ?? null,
43
+ distanceMetric: config.mode === 'off' ? null : 'cosine',
44
+ distanceCeiling: config.distanceCeiling ?? null,
45
+ backend: selectedBackend.id,
46
+ backendVersion: selectedBackend.version,
47
+ eligibleEntries: 0,
48
+ readyVectors: 0,
49
+ staleVectors: 0,
50
+ missingVectors: 0,
51
+ pendingJobs: 0,
52
+ leasedJobs: 0,
53
+ failedJobs: 0,
54
+ blockedJobs: 0,
55
+ coverageRatio: 0,
56
+ queryCacheRows: 0,
57
+ };
58
+ }
59
+ function activeStatus(database, config, active, backend) {
60
+ const selectedBackend = configuredBackend(config, backend);
61
+ const eligibleEntries = countRows(database, 'SELECT COUNT(*) AS count FROM entries');
62
+ if (active === null) {
63
+ return { ...emptyStatus(config, backend), eligibleEntries, missingVectors: eligibleEntries };
64
+ }
65
+ const readyVectors = countRows(database, `
66
+ SELECT COUNT(*) AS count
67
+ FROM entry_embeddings AS ee
68
+ JOIN entries AS e ON e.id = ee.entry_id
69
+ JOIN entry_revisions AS r ON r.entry_id = e.id AND r.revision = e.current_revision
70
+ WHERE ee.profile_id = ?
71
+ AND ee.revision = e.current_revision
72
+ AND ee.content_hash = r.content_hash
73
+ `, active.profileId);
74
+ const totalVectors = countRows(database, 'SELECT COUNT(*) AS count FROM entry_embeddings WHERE profile_id = ?', active.profileId);
75
+ const jobs = new Map(database.prepare(`
76
+ SELECT state, COUNT(*) AS count
77
+ FROM embedding_jobs
78
+ WHERE profile_id = ?
79
+ GROUP BY state
80
+ `).all(active.profileId).map((row) => [row.state, Number(row.count)]));
81
+ const selected = {
82
+ pending: jobs.get('pending') ?? 0,
83
+ leased: jobs.get('leased') ?? 0,
84
+ failed: jobs.get('failed') ?? 0,
85
+ blocked: jobs.get('blocked') ?? 0,
86
+ };
87
+ const queryCacheRows = countRows(database, 'SELECT COUNT(*) AS count FROM query_embeddings WHERE profile_id = ?', active.profileId);
88
+ return {
89
+ mode: config.mode,
90
+ activeProfileId: active.profileId,
91
+ providerKind: active.identity.providerKind,
92
+ model: active.identity.model,
93
+ dimensions: active.identity.dimensions,
94
+ distanceMetric: active.identity.distanceMetric,
95
+ distanceCeiling: active.identity.distanceCeiling,
96
+ backend: selectedBackend.id,
97
+ backendVersion: selectedBackend.version,
98
+ eligibleEntries,
99
+ readyVectors,
100
+ staleVectors: Math.max(0, totalVectors - readyVectors),
101
+ missingVectors: Math.max(0, eligibleEntries - readyVectors),
102
+ pendingJobs: selected.pending,
103
+ leasedJobs: selected.leased,
104
+ failedJobs: selected.failed,
105
+ blockedJobs: selected.blocked,
106
+ coverageRatio: eligibleEntries === 0 ? 0 : readyVectors / eligibleEntries,
107
+ queryCacheRows,
108
+ };
109
+ }
110
+ export function readEmbeddingStatus(database, config = parseEmbeddingConfig(), backend) {
111
+ const runtime = readEmbeddingRuntimeState(database);
112
+ const active = readActiveEmbeddingProfile(database);
113
+ if (active !== null && runtime.activeProfileId !== active.profile.profileId) {
114
+ throw new KiokukoError('INTEGRITY_ERROR', 'Embedding runtime active profile is inconsistent');
115
+ }
116
+ return activeStatus(database, config, active?.profile ?? null, backend);
117
+ }
118
+ function inspectVectors(database) {
119
+ let findings = 0;
120
+ const rows = database.prepare(`
121
+ SELECT entry_id, profile_id, dimensions, embedding, vector_hash
122
+ FROM entry_embeddings
123
+ ORDER BY profile_id ASC, entry_id ASC
124
+ `).all();
125
+ for (const row of rows) {
126
+ try {
127
+ if (typeof row.profile_id !== 'string' || typeof row.dimensions !== 'number' || typeof row.vector_hash !== 'string') {
128
+ findings += 1;
129
+ continue;
130
+ }
131
+ const bytes = row.embedding instanceof Uint8Array ? row.embedding : null;
132
+ if (bytes === null || bytes.byteLength !== row.dimensions * 4 || hashVectorBytes(bytes) !== row.vector_hash) {
133
+ findings += 1;
134
+ continue;
135
+ }
136
+ decodeVector(bytes, row.dimensions, row.vector_hash);
137
+ }
138
+ catch {
139
+ findings += 1;
140
+ }
141
+ }
142
+ return findings;
143
+ }
144
+ function inspectQueryCache(database) {
145
+ let findings = 0;
146
+ const rows = database.prepare(`
147
+ SELECT profile_id, dimensions, embedding, vector_hash
148
+ FROM query_embeddings
149
+ ORDER BY profile_id ASC, query_hash ASC
150
+ `).all();
151
+ for (const row of rows) {
152
+ try {
153
+ if (typeof row.dimensions !== 'number' || typeof row.vector_hash !== 'string' || !(row.embedding instanceof Uint8Array)) {
154
+ findings += 1;
155
+ continue;
156
+ }
157
+ decodeVector(row.embedding, row.dimensions, row.vector_hash);
158
+ }
159
+ catch {
160
+ findings += 1;
161
+ }
162
+ }
163
+ return findings;
164
+ }
165
+ function inspectJobReferences(database) {
166
+ return countRows(database, `
167
+ SELECT COUNT(*) AS count
168
+ FROM embedding_jobs AS j
169
+ LEFT JOIN entries AS e ON e.id = j.entry_id
170
+ LEFT JOIN entry_revisions AS r ON r.entry_id = j.entry_id AND r.revision = j.revision
171
+ WHERE e.id IS NULL OR r.entry_id IS NULL
172
+ OR (j.state = 'leased' AND (j.lease_id IS NULL OR j.lease_expires_at IS NULL))
173
+ OR (j.state <> 'leased' AND (j.lease_id IS NOT NULL OR j.lease_expires_at IS NOT NULL))
174
+ `);
175
+ }
176
+ function inspectConfiguration(database, config, active, backend) {
177
+ if (config.mode === 'off')
178
+ return 0;
179
+ let findings = 0;
180
+ if (active === null || !isEnabledEmbeddingConfig(config))
181
+ findings += 1;
182
+ else {
183
+ try {
184
+ if (embeddingProfileId(createEmbeddingProfile(config).identity) !== active.profile.profileId)
185
+ findings += 1;
186
+ }
187
+ catch {
188
+ findings += 1;
189
+ }
190
+ }
191
+ if (config.vectorBackend === 'sqlite-vec' && backend?.id !== 'sqlite-vec')
192
+ findings += 1;
193
+ return findings;
194
+ }
195
+ function safeConfig(environment) {
196
+ try {
197
+ return { config: parseEmbeddingConfig(environment), invalid: false };
198
+ }
199
+ catch {
200
+ return { invalid: true };
201
+ }
202
+ }
203
+ export function inspectEmbeddingHealth(database, environment = process.env, backend) {
204
+ const parsed = safeConfig(environment);
205
+ const config = parsed.config ?? { mode: 'off', provider: 'openai-compatible', allowRemote: false, vectorBackend: 'auto', timeoutMs: 30_000, batchSize: 16 };
206
+ let status = emptyStatus(config, backend);
207
+ let findings = parsed.invalid ? 1 : 0;
208
+ const tables = objectNames(database, 'table');
209
+ const indexes = objectNames(database, 'index');
210
+ const triggers = objectNames(database, 'trigger');
211
+ findings += EMBEDDING_TABLES.filter((name) => !tables.has(name)).length;
212
+ findings += EMBEDDING_INDEXES.filter((name) => !indexes.has(name)).length;
213
+ if (!triggers.has('embedding_profiles_immutable_update'))
214
+ findings += 1;
215
+ try {
216
+ const runtime = readEmbeddingRuntimeState(database);
217
+ const active = readActiveEmbeddingProfile(database);
218
+ status = activeStatus(database, config, active?.profile ?? null, backend);
219
+ findings += inspectConfiguration(database, config, active, backend);
220
+ findings += inspectVectors(database);
221
+ findings += inspectQueryCache(database);
222
+ findings += inspectJobReferences(database);
223
+ const runtimeRows = countRows(database, 'SELECT COUNT(*) AS count FROM embedding_runtime');
224
+ if (runtimeRows !== 1)
225
+ findings += 1;
226
+ const stale = countRows(database, `
227
+ SELECT COUNT(*) AS count
228
+ FROM entry_embeddings AS ee
229
+ JOIN entries AS e ON e.id = ee.entry_id
230
+ JOIN entry_revisions AS r ON r.entry_id = e.id AND r.revision = e.current_revision
231
+ WHERE ee.revision != e.current_revision OR ee.content_hash != r.content_hash
232
+ `);
233
+ if (stale !== status.staleVectors)
234
+ findings += 1;
235
+ }
236
+ catch {
237
+ findings += 1;
238
+ }
239
+ return {
240
+ status,
241
+ check: {
242
+ ok: findings === 0,
243
+ count: findings,
244
+ detail: parsed.invalid ? `findings=${findings}, configuration=invalid` : `findings=${findings}, mode=${config.mode}, backend=${status.backend ?? 'disabled'}`,
245
+ },
246
+ };
247
+ }
248
+ //# sourceMappingURL=diagnostics.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"diagnostics.js","sourceRoot":"","sources":["../../src/embedding/diagnostics.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,oBAAoB,EAAE,wBAAwB,EAAE,MAAM,aAAa,CAAC;AAC7E,OAAO,EAAE,kBAAkB,EAAE,sBAAsB,EAAE,MAAM,cAAc,CAAC;AAC1E,OAAO,EAAE,0BAA0B,EAAE,yBAAyB,EAAE,MAAM,YAAY,CAAC;AACnF,OAAO,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,aAAa,CAAC;AAG5D,MAAM,gBAAgB,GAAG;IACvB,oBAAoB;IACpB,mBAAmB;IACnB,kBAAkB;IAClB,gBAAgB;IAChB,kBAAkB;CACV,CAAC;AAEX,MAAM,iBAAiB,GAAG;IACxB,uCAAuC;IACvC,0BAA0B;IAC1B,0BAA0B;CAClB,CAAC;AAyCX,SAAS,SAAS,CAAC,QAAwB,EAAE,GAAW,EAAE,GAAG,UAAkC;IAC7F,OAAO,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,CAAC,GAAG,CAAoB,GAAG,UAAU,CAAC,EAAE,KAAK,IAAI,CAAC,CAAC,CAAC;AACzF,CAAC;AAED,SAAS,WAAW,CAAC,QAAwB,EAAE,IAAmC;IAChF,OAAO,IAAI,GAAG,CAAC,QAAQ,CAAC,OAAO,CAAC;;GAE/B,CAAC,CAAC,GAAG,CAAmB,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;AACzD,CAAC;AAED,SAAS,iBAAiB,CAAC,MAAuB,EAAE,OAAwC;IAC1F,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK;QAAE,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC9D,IAAI,OAAO,KAAK,SAAS;QAAE,OAAO,EAAE,EAAE,EAAE,OAAO,CAAC,EAAE,EAAE,OAAO,EAAE,OAAO,CAAC,EAAE,KAAK,YAAY,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7G,IAAI,MAAM,CAAC,aAAa,KAAK,YAAY;QAAE,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,OAAO,EAAE,QAAQ,EAAE,CAAC;IAC1F,OAAO,EAAE,EAAE,EAAE,YAAY,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;AAC7C,CAAC;AAED,SAAS,WAAW,CAAC,MAAuB,EAAE,OAA6B;IACzE,MAAM,eAAe,GAAG,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3D,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,eAAe,EAAE,IAAI;QACrB,YAAY,EAAE,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,QAAQ;QAC5D,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,IAAI;QAC3B,UAAU,EAAE,MAAM,CAAC,UAAU,IAAI,IAAI;QACrC,cAAc,EAAE,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,QAAQ;QACvD,eAAe,EAAE,MAAM,CAAC,eAAe,IAAI,IAAI;QAC/C,OAAO,EAAE,eAAe,CAAC,EAAE;QAC3B,cAAc,EAAE,eAAe,CAAC,OAAO;QACvC,eAAe,EAAE,CAAC;QAClB,YAAY,EAAE,CAAC;QACf,YAAY,EAAE,CAAC;QACf,cAAc,EAAE,CAAC;QACjB,WAAW,EAAE,CAAC;QACd,UAAU,EAAE,CAAC;QACb,UAAU,EAAE,CAAC;QACb,WAAW,EAAE,CAAC;QACd,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,CAAC;KAClB,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CACnB,QAAwB,EACxB,MAAuB,EACvB,MAA+B,EAC/B,OAA6B;IAE7B,MAAM,eAAe,GAAG,iBAAiB,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC3D,MAAM,eAAe,GAAG,SAAS,CAAC,QAAQ,EAAE,uCAAuC,CAAC,CAAC;IACrF,IAAI,MAAM,KAAK,IAAI,EAAE,CAAC;QACpB,OAAO,EAAE,GAAG,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,EAAE,eAAe,EAAE,cAAc,EAAE,eAAe,EAAE,CAAC;IAC/F,CAAC;IACD,MAAM,YAAY,GAAG,SAAS,CAAC,QAAQ,EAAE;;;;;;;;GAQxC,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IACrB,MAAM,YAAY,GAAG,SAAS,CAAC,QAAQ,EAAE,qEAAqE,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IAClI,MAAM,IAAI,GAAG,IAAI,GAAG,CAClB,QAAQ,CAAC,OAAO,CAAC;;;;;KAKhB,CAAC,CAAC,GAAG,CAAmC,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,MAAM,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,CAAC,CACxG,CAAC;IACF,MAAM,QAAQ,GAAG;QACf,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC;QACjC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC;QAC/B,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC;QAC/B,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,SAAS,CAAC,IAAI,CAAC;KAClC,CAAC;IACF,MAAM,cAAc,GAAG,SAAS,CAAC,QAAQ,EAAE,qEAAqE,EAAE,MAAM,CAAC,SAAS,CAAC,CAAC;IACpI,OAAO;QACL,IAAI,EAAE,MAAM,CAAC,IAAI;QACjB,eAAe,EAAE,MAAM,CAAC,SAAS;QACjC,YAAY,EAAE,MAAM,CAAC,QAAQ,CAAC,YAAY;QAC1C,KAAK,EAAE,MAAM,CAAC,QAAQ,CAAC,KAAK;QAC5B,UAAU,EAAE,MAAM,CAAC,QAAQ,CAAC,UAAU;QACtC,cAAc,EAAE,MAAM,CAAC,QAAQ,CAAC,cAAc;QAC9C,eAAe,EAAE,MAAM,CAAC,QAAQ,CAAC,eAAe;QAChD,OAAO,EAAE,eAAe,CAAC,EAAE;QAC3B,cAAc,EAAE,eAAe,CAAC,OAAO;QACvC,eAAe;QACf,YAAY;QACZ,YAAY,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,YAAY,GAAG,YAAY,CAAC;QACtD,cAAc,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,eAAe,GAAG,YAAY,CAAC;QAC3D,WAAW,EAAE,QAAQ,CAAC,OAAO;QAC7B,UAAU,EAAE,QAAQ,CAAC,MAAM;QAC3B,UAAU,EAAE,QAAQ,CAAC,MAAM;QAC3B,WAAW,EAAE,QAAQ,CAAC,OAAO;QAC7B,aAAa,EAAE,eAAe,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,eAAe;QACzE,cAAc;KACf,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CACjC,QAAwB,EACxB,SAA0B,oBAAoB,EAAE,EAChD,OAA6B;IAE7B,MAAM,OAAO,GAAG,yBAAyB,CAAC,QAAQ,CAAC,CAAC;IACpD,MAAM,MAAM,GAAG,0BAA0B,CAAC,QAAQ,CAAC,CAAC;IACpD,IAAI,MAAM,KAAK,IAAI,IAAI,OAAO,CAAC,eAAe,KAAK,MAAM,CAAC,OAAO,CAAC,SAAS,EAAE,CAAC;QAC5E,MAAM,IAAI,YAAY,CAAC,iBAAiB,EAAE,kDAAkD,CAAC,CAAC;IAChG,CAAC;IACD,OAAO,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,IAAI,IAAI,EAAE,OAAO,CAAC,CAAC;AAC1E,CAAC;AAED,SAAS,cAAc,CAAC,QAAwB;IAC9C,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC;;;;GAI7B,CAAC,CAAC,GAAG,EAAa,CAAC;IACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,EAAE,CAAC;gBACpH,QAAQ,IAAI,CAAC,CAAC;gBACd,SAAS;YACX,CAAC;YACD,MAAM,KAAK,GAAG,GAAG,CAAC,SAAS,YAAY,UAAU,CAAC,CAAC,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC;YACzE,IAAI,KAAK,KAAK,IAAI,IAAI,KAAK,CAAC,UAAU,KAAK,GAAG,CAAC,UAAU,GAAG,CAAC,IAAI,eAAe,CAAC,KAAK,CAAC,KAAK,GAAG,CAAC,WAAW,EAAE,CAAC;gBAC5G,QAAQ,IAAI,CAAC,CAAC;gBACd,SAAS;YACX,CAAC;YACD,YAAY,CAAC,KAAK,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;QACvD,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,IAAI,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,iBAAiB,CAAC,QAAwB;IACjD,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,MAAM,IAAI,GAAG,QAAQ,CAAC,OAAO,CAAC;;;;GAI7B,CAAC,CAAC,GAAG,EAAa,CAAC;IACpB,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,CAAC;YACH,IAAI,OAAO,GAAG,CAAC,UAAU,KAAK,QAAQ,IAAI,OAAO,GAAG,CAAC,WAAW,KAAK,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,SAAS,YAAY,UAAU,CAAC,EAAE,CAAC;gBACxH,QAAQ,IAAI,CAAC,CAAC;gBACd,SAAS;YACX,CAAC;YACD,YAAY,CAAC,GAAG,CAAC,SAAS,EAAE,GAAG,CAAC,UAAU,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;QAC/D,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,IAAI,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;IACD,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,oBAAoB,CAAC,QAAwB;IACpD,OAAO,SAAS,CAAC,QAAQ,EAAE;;;;;;;;GAQ1B,CAAC,CAAC;AACL,CAAC;AAED,SAAS,oBAAoB,CAC3B,QAAwB,EACxB,MAAuB,EACvB,MAAqD,EACrD,OAAwC;IAExC,IAAI,MAAM,CAAC,IAAI,KAAK,KAAK;QAAE,OAAO,CAAC,CAAC;IACpC,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,MAAM,KAAK,IAAI,IAAI,CAAC,wBAAwB,CAAC,MAAM,CAAC;QAAE,QAAQ,IAAI,CAAC,CAAC;SACnE,CAAC;QACJ,IAAI,CAAC;YACH,IAAI,kBAAkB,CAAC,sBAAsB,CAAC,MAAM,CAAC,CAAC,QAAQ,CAAC,KAAK,MAAM,CAAC,OAAO,CAAC,SAAS;gBAAE,QAAQ,IAAI,CAAC,CAAC;QAC9G,CAAC;QAAC,MAAM,CAAC;YACP,QAAQ,IAAI,CAAC,CAAC;QAChB,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,aAAa,KAAK,YAAY,IAAI,OAAO,EAAE,EAAE,KAAK,YAAY;QAAE,QAAQ,IAAI,CAAC,CAAC;IACzF,OAAO,QAAQ,CAAC;AAClB,CAAC;AAED,SAAS,UAAU,CAAC,WAA8B;IAChD,IAAI,CAAC;QACH,OAAO,EAAE,MAAM,EAAE,oBAAoB,CAAC,WAAW,CAAC,EAAE,OAAO,EAAE,KAAK,EAAE,CAAC;IACvE,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC;IAC3B,CAAC;AACH,CAAC;AAED,MAAM,UAAU,sBAAsB,CACpC,QAAwB,EACxB,cAAiC,OAAO,CAAC,GAAG,EAC5C,OAA6B;IAE7B,MAAM,MAAM,GAAG,UAAU,CAAC,WAAW,CAAC,CAAC;IACvC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,mBAAmB,EAAE,WAAW,EAAE,KAAK,EAAE,aAAa,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,SAAS,EAAE,EAAE,EAAE,CAAC;IAC5J,IAAI,MAAM,GAAG,WAAW,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAC1C,IAAI,QAAQ,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC;IACtC,MAAM,MAAM,GAAG,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC9C,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,EAAE,OAAO,CAAC,CAAC;IAC/C,MAAM,QAAQ,GAAG,WAAW,CAAC,QAAQ,EAAE,SAAS,CAAC,CAAC;IAClD,QAAQ,IAAI,gBAAgB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,MAAM,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IACxE,QAAQ,IAAI,iBAAiB,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,OAAO,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC;IAC1E,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,qCAAqC,CAAC;QAAE,QAAQ,IAAI,CAAC,CAAC;IAExE,IAAI,CAAC;QACH,MAAM,OAAO,GAAG,yBAAyB,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,MAAM,GAAG,0BAA0B,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,GAAG,YAAY,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,IAAI,IAAI,EAAE,OAAO,CAAC,CAAC;QAC1E,QAAQ,IAAI,oBAAoB,CAAC,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,CAAC,CAAC;QACpE,QAAQ,IAAI,cAAc,CAAC,QAAQ,CAAC,CAAC;QACrC,QAAQ,IAAI,iBAAiB,CAAC,QAAQ,CAAC,CAAC;QACxC,QAAQ,IAAI,oBAAoB,CAAC,QAAQ,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,SAAS,CAAC,QAAQ,EAAE,iDAAiD,CAAC,CAAC;QAC3F,IAAI,WAAW,KAAK,CAAC;YAAE,QAAQ,IAAI,CAAC,CAAC;QACrC,MAAM,KAAK,GAAG,SAAS,CAAC,QAAQ,EAAE;;;;;;KAMjC,CAAC,CAAC;QACH,IAAI,KAAK,KAAK,MAAM,CAAC,YAAY;YAAE,QAAQ,IAAI,CAAC,CAAC;IACnD,CAAC;IAAC,MAAM,CAAC;QACP,QAAQ,IAAI,CAAC,CAAC;IAChB,CAAC;IAED,OAAO;QACL,MAAM;QACN,KAAK,EAAE;YACL,EAAE,EAAE,QAAQ,KAAK,CAAC;YAClB,KAAK,EAAE,QAAQ;YACf,MAAM,EAAE,MAAM,CAAC,OAAO,CAAC,CAAC,CAAC,YAAY,QAAQ,yBAAyB,CAAC,CAAC,CAAC,YAAY,QAAQ,UAAU,MAAM,CAAC,IAAI,aAAa,MAAM,CAAC,OAAO,IAAI,UAAU,EAAE;SAC9J;KACF,CAAC;AACJ,CAAC"}
@@ -0,0 +1,15 @@
1
+ import { type JsonObject, type EntryKind } from '../serialization/validate.js';
2
+ import type { EmbeddingDocument } from './types.js';
3
+ export declare const EMBEDDING_DOCUMENT_TEMPLATE_VERSION: 1;
4
+ export declare const MAX_EMBEDDING_DOCUMENT_BYTES: number;
5
+ export declare const BODY_TRUNCATION_MARKER: "[body truncated]";
6
+ export interface EmbeddingDocumentInput {
7
+ readonly kind: EntryKind;
8
+ readonly title: string;
9
+ readonly summary: string | null;
10
+ readonly body: string;
11
+ readonly tags: readonly string[];
12
+ readonly scope: JsonObject;
13
+ }
14
+ export declare function buildEmbeddingDocument(input: EmbeddingDocumentInput): EmbeddingDocument;
15
+ //# sourceMappingURL=document.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"document.d.ts","sourceRoot":"","sources":["../../src/embedding/document.ts"],"names":[],"mappings":"AAGA,OAAO,EAAoC,KAAK,UAAU,EAAE,KAAK,SAAS,EAAE,MAAM,8BAA8B,CAAC;AAGjH,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEpD,eAAO,MAAM,mCAAmC,EAAG,CAAU,CAAC;AAC9D,eAAO,MAAM,4BAA4B,QAAY,CAAC;AACtD,eAAO,MAAM,sBAAsB,EAAG,kBAA2B,CAAC;AAElE,MAAM,WAAW,sBAAsB;IACrC,QAAQ,CAAC,IAAI,EAAE,SAAS,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,IAAI,EAAE,SAAS,MAAM,EAAE,CAAC;IACjC,QAAQ,CAAC,KAAK,EAAE,UAAU,CAAC;CAC5B;AAmDD,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,sBAAsB,GAAG,iBAAiB,CA6CvF"}
@@ -0,0 +1,102 @@
1
+ import { createHash } from 'node:crypto';
2
+ import { TextDecoder, TextEncoder } from 'node:util';
3
+ import { KiokukoError } from '../errors.js';
4
+ import { canonicalJson, canonicalTagOrder } from '../serialization/validate.js';
5
+ import { findSecret } from '../memory/secrets.js';
6
+ import { validateApplicability, validateSignals } from '../memory/structured-memory.js';
7
+ export const EMBEDDING_DOCUMENT_TEMPLATE_VERSION = 1;
8
+ export const MAX_EMBEDDING_DOCUMENT_BYTES = 32 * 1024;
9
+ export const BODY_TRUNCATION_MARKER = '[body truncated]';
10
+ const encoder = new TextEncoder();
11
+ const decoder = new TextDecoder('utf-8', { fatal: true });
12
+ function invalid(message) {
13
+ throw new KiokukoError('VALIDATION_ERROR', message);
14
+ }
15
+ function normalizeText(value, field) {
16
+ const normalized = value.normalize('NFKC').replaceAll('\r\n', '\n').replaceAll('\r', '\n');
17
+ if (/[\u0000-\u0008\u000b\u000c\u000e-\u001f\u007f]/u.test(normalized))
18
+ invalid(`${field} contains a forbidden control character`);
19
+ return normalized;
20
+ }
21
+ function normalizedTagList(tags) {
22
+ for (const tag of tags) {
23
+ if (typeof tag !== 'string' || tag.trim().length === 0 || tag.length > 200)
24
+ invalid('Embedding document tags are invalid');
25
+ }
26
+ return canonicalTagOrder(tags.map((tag) => normalizeText(tag, 'tag')));
27
+ }
28
+ function structuredScope(scope) {
29
+ const candidate = scope;
30
+ if (candidate.schemaVersion !== 2 && candidate.schemaVersion !== 3)
31
+ return { applicability: {}, signals: {} };
32
+ const applicability = candidate.applicability === undefined ? {} : validateApplicability(candidate.applicability);
33
+ const signals = candidate.signals === undefined ? {} : validateSignals(candidate.signals);
34
+ return {
35
+ applicability: applicability,
36
+ signals: signals,
37
+ };
38
+ }
39
+ function truncateUtf8(value, maxBytes) {
40
+ if (encoder.encode(value).byteLength <= maxBytes)
41
+ return value;
42
+ const bytes = encoder.encode(value);
43
+ let end = Math.min(bytes.byteLength, maxBytes);
44
+ while (end > 0) {
45
+ try {
46
+ return decoder.decode(bytes.subarray(0, end));
47
+ }
48
+ catch {
49
+ end -= 1;
50
+ }
51
+ }
52
+ return '';
53
+ }
54
+ function documentHash(bytes) {
55
+ return createHash('sha256').update(bytes).digest('hex');
56
+ }
57
+ export function buildEmbeddingDocument(input) {
58
+ if (input.kind !== 'fact' && input.kind !== 'decision' && input.kind !== 'lesson' && input.kind !== 'preference' && input.kind !== 'reference') {
59
+ invalid('Embedding document kind is invalid');
60
+ }
61
+ const title = normalizeText(input.title, 'title').trim();
62
+ const summary = input.summary === null ? '' : normalizeText(input.summary, 'summary').trim();
63
+ const body = normalizeText(input.body, 'body');
64
+ const tags = normalizedTagList(input.tags);
65
+ const { applicability, signals } = structuredScope(input.scope);
66
+ const metadataText = [
67
+ 'kiokuko-memory-v1',
68
+ `kind: ${input.kind}`,
69
+ `title: ${title}`,
70
+ `summary: ${summary}`,
71
+ 'tags:',
72
+ ...tags.map((tag) => `- ${tag}`),
73
+ 'applicability:',
74
+ canonicalJson(applicability),
75
+ 'signals:',
76
+ canonicalJson(signals),
77
+ 'body:',
78
+ ].join('\n') + '\n';
79
+ const metadataBytes = encoder.encode(metadataText);
80
+ if (metadataBytes.byteLength >= MAX_EMBEDDING_DOCUMENT_BYTES) {
81
+ invalid('Embedding document metadata exceeds the byte limit');
82
+ }
83
+ const bodyBytes = encoder.encode(body);
84
+ let text = metadataText + body;
85
+ let truncated = false;
86
+ if (metadataBytes.byteLength + bodyBytes.byteLength > MAX_EMBEDDING_DOCUMENT_BYTES) {
87
+ const marker = `\n${BODY_TRUNCATION_MARKER}`;
88
+ const markerBytes = encoder.encode(marker);
89
+ const available = MAX_EMBEDDING_DOCUMENT_BYTES - metadataBytes.byteLength - markerBytes.byteLength;
90
+ if (available < 0)
91
+ invalid('Embedding document metadata exceeds the byte limit');
92
+ text = metadataText + truncateUtf8(body, available) + marker;
93
+ truncated = true;
94
+ }
95
+ const bytes = encoder.encode(text);
96
+ const secretFinding = findSecret(text);
97
+ if (secretFinding !== undefined) {
98
+ throw new KiokukoError('SECURITY_REJECTION', 'Embedding document resembles a secret and was not sent');
99
+ }
100
+ return Object.freeze({ text, bytes, documentHash: documentHash(bytes), truncated });
101
+ }
102
+ //# sourceMappingURL=document.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"document.js","sourceRoot":"","sources":["../../src/embedding/document.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AACzC,OAAO,EAAE,WAAW,EAAE,WAAW,EAAE,MAAM,WAAW,CAAC;AACrD,OAAO,EAAE,YAAY,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,EAAE,aAAa,EAAE,iBAAiB,EAAmC,MAAM,8BAA8B,CAAC;AACjH,OAAO,EAAE,UAAU,EAAE,MAAM,sBAAsB,CAAC;AAClD,OAAO,EAAE,qBAAqB,EAAE,eAAe,EAAE,MAAM,gCAAgC,CAAC;AAGxF,MAAM,CAAC,MAAM,mCAAmC,GAAG,CAAU,CAAC;AAC9D,MAAM,CAAC,MAAM,4BAA4B,GAAG,EAAE,GAAG,IAAI,CAAC;AACtD,MAAM,CAAC,MAAM,sBAAsB,GAAG,kBAA2B,CAAC;AAWlE,MAAM,OAAO,GAAG,IAAI,WAAW,EAAE,CAAC;AAClC,MAAM,OAAO,GAAG,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;AAE1D,SAAS,OAAO,CAAC,OAAe;IAC9B,MAAM,IAAI,YAAY,CAAC,kBAAkB,EAAE,OAAO,CAAC,CAAC;AACtD,CAAC;AAED,SAAS,aAAa,CAAC,KAAa,EAAE,KAAa;IACjD,MAAM,UAAU,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,UAAU,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC,UAAU,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IAC3F,IAAI,iDAAiD,CAAC,IAAI,CAAC,UAAU,CAAC;QAAE,OAAO,CAAC,GAAG,KAAK,yCAAyC,CAAC,CAAC;IACnI,OAAO,UAAU,CAAC;AACpB,CAAC;AAED,SAAS,iBAAiB,CAAC,IAAuB;IAChD,KAAK,MAAM,GAAG,IAAI,IAAI,EAAE,CAAC;QACvB,IAAI,OAAO,GAAG,KAAK,QAAQ,IAAI,GAAG,CAAC,IAAI,EAAE,CAAC,MAAM,KAAK,CAAC,IAAI,GAAG,CAAC,MAAM,GAAG,GAAG;YAAE,OAAO,CAAC,qCAAqC,CAAC,CAAC;IAC7H,CAAC;IACD,OAAO,iBAAiB,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,aAAa,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC;AACzE,CAAC;AAED,SAAS,eAAe,CAAC,KAAiB;IACxC,MAAM,SAAS,GAAG,KAAgC,CAAC;IACnD,IAAI,SAAS,CAAC,aAAa,KAAK,CAAC,IAAI,SAAS,CAAC,aAAa,KAAK,CAAC;QAAE,OAAO,EAAE,aAAa,EAAE,EAAE,EAAE,OAAO,EAAE,EAAE,EAAE,CAAC;IAC9G,MAAM,aAAa,GAAG,SAAS,CAAC,aAAa,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,qBAAqB,CAAC,SAAS,CAAC,aAAa,CAAC,CAAC;IAClH,MAAM,OAAO,GAAG,SAAS,CAAC,OAAO,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IAC1F,OAAO;QACL,aAAa,EAAE,aAAsC;QACrD,OAAO,EAAE,OAAgC;KAC1C,CAAC;AACJ,CAAC;AAED,SAAS,YAAY,CAAC,KAAa,EAAE,QAAgB;IACnD,IAAI,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,UAAU,IAAI,QAAQ;QAAE,OAAO,KAAK,CAAC;IAC/D,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACpC,IAAI,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IAC/C,OAAO,GAAG,GAAG,CAAC,EAAE,CAAC;QACf,IAAI,CAAC;YACH,OAAO,OAAO,CAAC,MAAM,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;QAChD,CAAC;QAAC,MAAM,CAAC;YACP,GAAG,IAAI,CAAC,CAAC;QACX,CAAC;IACH,CAAC;IACD,OAAO,EAAE,CAAC;AACZ,CAAC;AAED,SAAS,YAAY,CAAC,KAAiB;IACrC,OAAO,UAAU,CAAC,QAAQ,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;AAC1D,CAAC;AAED,MAAM,UAAU,sBAAsB,CAAC,KAA6B;IAClE,IAAI,KAAK,CAAC,IAAI,KAAK,MAAM,IAAI,KAAK,CAAC,IAAI,KAAK,UAAU,IAAI,KAAK,CAAC,IAAI,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,KAAK,YAAY,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC/I,OAAO,CAAC,oCAAoC,CAAC,CAAC;IAChD,CAAC;IACD,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,EAAE,OAAO,CAAC,CAAC,IAAI,EAAE,CAAC;IACzD,MAAM,OAAO,GAAG,KAAK,CAAC,OAAO,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC,IAAI,EAAE,CAAC;IAC7F,MAAM,IAAI,GAAG,aAAa,CAAC,KAAK,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;IAC/C,MAAM,IAAI,GAAG,iBAAiB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC3C,MAAM,EAAE,aAAa,EAAE,OAAO,EAAE,GAAG,eAAe,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChE,MAAM,YAAY,GAAG;QACnB,mBAAmB;QACnB,SAAS,KAAK,CAAC,IAAI,EAAE;QACrB,UAAU,KAAK,EAAE;QACjB,YAAY,OAAO,EAAE;QACrB,OAAO;QACP,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,KAAK,GAAG,EAAE,CAAC;QAChC,gBAAgB;QAChB,aAAa,CAAC,aAAa,CAAC;QAC5B,UAAU;QACV,aAAa,CAAC,OAAO,CAAC;QACtB,OAAO;KACR,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;IACpB,MAAM,aAAa,GAAG,OAAO,CAAC,MAAM,CAAC,YAAY,CAAC,CAAC;IACnD,IAAI,aAAa,CAAC,UAAU,IAAI,4BAA4B,EAAE,CAAC;QAC7D,OAAO,CAAC,oDAAoD,CAAC,CAAC;IAChE,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,IAAI,GAAG,YAAY,GAAG,IAAI,CAAC;IAC/B,IAAI,SAAS,GAAG,KAAK,CAAC;IACtB,IAAI,aAAa,CAAC,UAAU,GAAG,SAAS,CAAC,UAAU,GAAG,4BAA4B,EAAE,CAAC;QACnF,MAAM,MAAM,GAAG,KAAK,sBAAsB,EAAE,CAAC;QAC7C,MAAM,WAAW,GAAG,OAAO,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAC3C,MAAM,SAAS,GAAG,4BAA4B,GAAG,aAAa,CAAC,UAAU,GAAG,WAAW,CAAC,UAAU,CAAC;QACnG,IAAI,SAAS,GAAG,CAAC;YAAE,OAAO,CAAC,oDAAoD,CAAC,CAAC;QACjF,IAAI,GAAG,YAAY,GAAG,YAAY,CAAC,IAAI,EAAE,SAAS,CAAC,GAAG,MAAM,CAAC;QAC7D,SAAS,GAAG,IAAI,CAAC;IACnB,CAAC;IAED,MAAM,KAAK,GAAG,OAAO,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACnC,MAAM,aAAa,GAAG,UAAU,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,aAAa,KAAK,SAAS,EAAE,CAAC;QAChC,MAAM,IAAI,YAAY,CAAC,oBAAoB,EAAE,wDAAwD,CAAC,CAAC;IACzG,CAAC;IACD,OAAO,MAAM,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,YAAY,EAAE,YAAY,CAAC,KAAK,CAAC,EAAE,SAAS,EAAE,CAAC,CAAC;AACtF,CAAC"}
@@ -0,0 +1,11 @@
1
+ import type { SqliteDatabase } from '../db/adapter.js';
2
+ import type { VectorHit, VectorSearchBackend, VectorSearchInput } from './types.js';
3
+ export declare const JAVASCRIPT_BACKEND_ID = "javascript";
4
+ export declare const MAX_JAVASCRIPT_BACKEND_ENTRIES = 10000;
5
+ export declare const MAX_VECTOR_SEARCH_LIMIT = 120;
6
+ export declare class JavaScriptVectorSearchBackend implements VectorSearchBackend {
7
+ readonly id = "javascript";
8
+ search(database: SqliteDatabase, input: VectorSearchInput): VectorHit[];
9
+ }
10
+ export declare const createJavaScriptVectorSearchBackend: () => VectorSearchBackend;
11
+ //# sourceMappingURL=javascript-backend.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"javascript-backend.d.ts","sourceRoot":"","sources":["../../src/embedding/javascript-backend.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAa,MAAM,kBAAkB,CAAC;AAIlE,OAAO,KAAK,EAAE,SAAS,EAAE,mBAAmB,EAAE,iBAAiB,EAAE,MAAM,YAAY,CAAC;AAEpF,eAAO,MAAM,qBAAqB,eAAe,CAAC;AAClD,eAAO,MAAM,8BAA8B,QAAS,CAAC;AACrD,eAAO,MAAM,uBAAuB,MAAM,CAAC;AAqH3C,qBAAa,6BAA8B,YAAW,mBAAmB;IACvE,QAAQ,CAAC,EAAE,gBAAyB;IAEpC,MAAM,CAAC,QAAQ,EAAE,cAAc,EAAE,KAAK,EAAE,iBAAiB,GAAG,SAAS,EAAE;CAgDxE;AAED,eAAO,MAAM,mCAAmC,QAAO,mBAA0D,CAAC"}