@tenphi/akno-core 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (485) hide show
  1. package/LICENSE +83 -0
  2. package/README.md +26 -0
  3. package/config/default.jsonc +548 -0
  4. package/dist/bench/answer-corpus.d.ts +48 -0
  5. package/dist/bench/answer-corpus.d.ts.map +1 -0
  6. package/dist/bench/answer-corpus.js +491 -0
  7. package/dist/bench/answer-corpus.js.map +1 -0
  8. package/dist/bench/answer.d.ts +156 -0
  9. package/dist/bench/answer.d.ts.map +1 -0
  10. package/dist/bench/answer.js +608 -0
  11. package/dist/bench/answer.js.map +1 -0
  12. package/dist/bench/auto-recall-answer-corpus.d.ts +21 -0
  13. package/dist/bench/auto-recall-answer-corpus.d.ts.map +1 -0
  14. package/dist/bench/auto-recall-answer-corpus.js +245 -0
  15. package/dist/bench/auto-recall-answer-corpus.js.map +1 -0
  16. package/dist/bench/auto-recall-answer.d.ts +198 -0
  17. package/dist/bench/auto-recall-answer.d.ts.map +1 -0
  18. package/dist/bench/auto-recall-answer.js +720 -0
  19. package/dist/bench/auto-recall-answer.js.map +1 -0
  20. package/dist/bench/auto-recall-corpus.d.ts +45 -0
  21. package/dist/bench/auto-recall-corpus.d.ts.map +1 -0
  22. package/dist/bench/auto-recall-corpus.js +221 -0
  23. package/dist/bench/auto-recall-corpus.js.map +1 -0
  24. package/dist/bench/auto-recall.d.ts +171 -0
  25. package/dist/bench/auto-recall.d.ts.map +1 -0
  26. package/dist/bench/auto-recall.js +651 -0
  27. package/dist/bench/auto-recall.js.map +1 -0
  28. package/dist/bench/entity-resolution.d.ts +37 -0
  29. package/dist/bench/entity-resolution.d.ts.map +1 -0
  30. package/dist/bench/entity-resolution.js +121 -0
  31. package/dist/bench/entity-resolution.js.map +1 -0
  32. package/dist/bench/graph.d.ts +57 -0
  33. package/dist/bench/graph.d.ts.map +1 -0
  34. package/dist/bench/graph.js +573 -0
  35. package/dist/bench/graph.js.map +1 -0
  36. package/dist/bench/llm-ranking-probe.d.ts +27 -0
  37. package/dist/bench/llm-ranking-probe.d.ts.map +1 -0
  38. package/dist/bench/llm-ranking-probe.js +105 -0
  39. package/dist/bench/llm-ranking-probe.js.map +1 -0
  40. package/dist/bench/merge-discovery-corpus.d.ts +26 -0
  41. package/dist/bench/merge-discovery-corpus.d.ts.map +1 -0
  42. package/dist/bench/merge-discovery-corpus.js +114 -0
  43. package/dist/bench/merge-discovery-corpus.js.map +1 -0
  44. package/dist/bench/merge-discovery-review.d.ts +81 -0
  45. package/dist/bench/merge-discovery-review.d.ts.map +1 -0
  46. package/dist/bench/merge-discovery-review.js +149 -0
  47. package/dist/bench/merge-discovery-review.js.map +1 -0
  48. package/dist/bench/merge-discovery.d.ts +122 -0
  49. package/dist/bench/merge-discovery.d.ts.map +1 -0
  50. package/dist/bench/merge-discovery.js +390 -0
  51. package/dist/bench/merge-discovery.js.map +1 -0
  52. package/dist/bench/mixed-retrieval.d.ts +35 -0
  53. package/dist/bench/mixed-retrieval.d.ts.map +1 -0
  54. package/dist/bench/mixed-retrieval.js +355 -0
  55. package/dist/bench/mixed-retrieval.js.map +1 -0
  56. package/dist/bench/ranking-corpus.d.ts +29 -0
  57. package/dist/bench/ranking-corpus.d.ts.map +1 -0
  58. package/dist/bench/ranking-corpus.js +514 -0
  59. package/dist/bench/ranking-corpus.js.map +1 -0
  60. package/dist/bench/ranking-end-to-end.d.ts +112 -0
  61. package/dist/bench/ranking-end-to-end.d.ts.map +1 -0
  62. package/dist/bench/ranking-end-to-end.js +479 -0
  63. package/dist/bench/ranking-end-to-end.js.map +1 -0
  64. package/dist/bench/ranking-latency.d.ts +71 -0
  65. package/dist/bench/ranking-latency.d.ts.map +1 -0
  66. package/dist/bench/ranking-latency.js +130 -0
  67. package/dist/bench/ranking-latency.js.map +1 -0
  68. package/dist/bench/ranking-matrix.d.ts +141 -0
  69. package/dist/bench/ranking-matrix.d.ts.map +1 -0
  70. package/dist/bench/ranking-matrix.js +526 -0
  71. package/dist/bench/ranking-matrix.js.map +1 -0
  72. package/dist/bench/ranking-review.d.ts +93 -0
  73. package/dist/bench/ranking-review.d.ts.map +1 -0
  74. package/dist/bench/ranking-review.js +249 -0
  75. package/dist/bench/ranking-review.js.map +1 -0
  76. package/dist/bench/ranking.d.ts +122 -0
  77. package/dist/bench/ranking.d.ts.map +1 -0
  78. package/dist/bench/ranking.js +547 -0
  79. package/dist/bench/ranking.js.map +1 -0
  80. package/dist/bench.d.ts +60 -0
  81. package/dist/bench.d.ts.map +1 -0
  82. package/dist/bench.js +167 -0
  83. package/dist/bench.js.map +1 -0
  84. package/dist/config/jsonc.d.ts +8 -0
  85. package/dist/config/jsonc.d.ts.map +1 -0
  86. package/dist/config/jsonc.js +85 -0
  87. package/dist/config/jsonc.js.map +1 -0
  88. package/dist/config/load.d.ts +43 -0
  89. package/dist/config/load.d.ts.map +1 -0
  90. package/dist/config/load.js +608 -0
  91. package/dist/config/load.js.map +1 -0
  92. package/dist/config/paths.d.ts +10 -0
  93. package/dist/config/paths.d.ts.map +1 -0
  94. package/dist/config/paths.js +38 -0
  95. package/dist/config/paths.js.map +1 -0
  96. package/dist/config/schema.d.ts +698 -0
  97. package/dist/config/schema.d.ts.map +1 -0
  98. package/dist/config/schema.js +394 -0
  99. package/dist/config/schema.js.map +1 -0
  100. package/dist/config/write-rules.d.ts +41 -0
  101. package/dist/config/write-rules.d.ts.map +1 -0
  102. package/dist/config/write-rules.js +225 -0
  103. package/dist/config/write-rules.js.map +1 -0
  104. package/dist/config/write-setup.d.ts +42 -0
  105. package/dist/config/write-setup.d.ts.map +1 -0
  106. package/dist/config/write-setup.js +358 -0
  107. package/dist/config/write-setup.js.map +1 -0
  108. package/dist/context.d.ts +64 -0
  109. package/dist/context.d.ts.map +1 -0
  110. package/dist/context.js +14 -0
  111. package/dist/context.js.map +1 -0
  112. package/dist/doctor.d.ts +115 -0
  113. package/dist/doctor.d.ts.map +1 -0
  114. package/dist/doctor.js +388 -0
  115. package/dist/doctor.js.map +1 -0
  116. package/dist/index/chunk.d.ts +62 -0
  117. package/dist/index/chunk.d.ts.map +1 -0
  118. package/dist/index/chunk.js +307 -0
  119. package/dist/index/chunk.js.map +1 -0
  120. package/dist/index/defer.d.ts +32 -0
  121. package/dist/index/defer.d.ts.map +1 -0
  122. package/dist/index/defer.js +66 -0
  123. package/dist/index/defer.js.map +1 -0
  124. package/dist/index/derive.d.ts +99 -0
  125. package/dist/index/derive.d.ts.map +1 -0
  126. package/dist/index/derive.js +378 -0
  127. package/dist/index/derive.js.map +1 -0
  128. package/dist/index/entity-resolution.d.ts +68 -0
  129. package/dist/index/entity-resolution.d.ts.map +1 -0
  130. package/dist/index/entity-resolution.js +305 -0
  131. package/dist/index/entity-resolution.js.map +1 -0
  132. package/dist/index/graph.d.ts +46 -0
  133. package/dist/index/graph.d.ts.map +1 -0
  134. package/dist/index/graph.js +618 -0
  135. package/dist/index/graph.js.map +1 -0
  136. package/dist/index/indexer.d.ts +301 -0
  137. package/dist/index/indexer.d.ts.map +1 -0
  138. package/dist/index/indexer.js +1398 -0
  139. package/dist/index/indexer.js.map +1 -0
  140. package/dist/index/revision-barrier.d.ts +30 -0
  141. package/dist/index/revision-barrier.d.ts.map +1 -0
  142. package/dist/index/revision-barrier.js +140 -0
  143. package/dist/index/revision-barrier.js.map +1 -0
  144. package/dist/index.d.ts +55 -0
  145. package/dist/index.d.ts.map +1 -0
  146. package/dist/index.js +37 -0
  147. package/dist/index.js.map +1 -0
  148. package/dist/ingest/adoption-eligibility.d.ts +4 -0
  149. package/dist/ingest/adoption-eligibility.d.ts.map +1 -0
  150. package/dist/ingest/adoption-eligibility.js +18 -0
  151. package/dist/ingest/adoption-eligibility.js.map +1 -0
  152. package/dist/ingest/availability.d.ts +15 -0
  153. package/dist/ingest/availability.d.ts.map +1 -0
  154. package/dist/ingest/availability.js +37 -0
  155. package/dist/ingest/availability.js.map +1 -0
  156. package/dist/ingest/extract.d.ts +58 -0
  157. package/dist/ingest/extract.d.ts.map +1 -0
  158. package/dist/ingest/extract.js +255 -0
  159. package/dist/ingest/extract.js.map +1 -0
  160. package/dist/ingest/fetch.d.ts +35 -0
  161. package/dist/ingest/fetch.d.ts.map +1 -0
  162. package/dist/ingest/fetch.js +142 -0
  163. package/dist/ingest/fetch.js.map +1 -0
  164. package/dist/ingest/inbox.d.ts +53 -0
  165. package/dist/ingest/inbox.d.ts.map +1 -0
  166. package/dist/ingest/inbox.js +110 -0
  167. package/dist/ingest/inbox.js.map +1 -0
  168. package/dist/ingest/name.d.ts +72 -0
  169. package/dist/ingest/name.d.ts.map +1 -0
  170. package/dist/ingest/name.js +208 -0
  171. package/dist/ingest/name.js.map +1 -0
  172. package/dist/ingest/parts.d.ts +78 -0
  173. package/dist/ingest/parts.d.ts.map +1 -0
  174. package/dist/ingest/parts.js +52 -0
  175. package/dist/ingest/parts.js.map +1 -0
  176. package/dist/ingest/rendition.d.ts +75 -0
  177. package/dist/ingest/rendition.d.ts.map +1 -0
  178. package/dist/ingest/rendition.js +92 -0
  179. package/dist/ingest/rendition.js.map +1 -0
  180. package/dist/ingest/store.d.ts +66 -0
  181. package/dist/ingest/store.d.ts.map +1 -0
  182. package/dist/ingest/store.js +92 -0
  183. package/dist/ingest/store.js.map +1 -0
  184. package/dist/kb/folders.d.ts +37 -0
  185. package/dist/kb/folders.d.ts.map +1 -0
  186. package/dist/kb/folders.js +133 -0
  187. package/dist/kb/folders.js.map +1 -0
  188. package/dist/kb/frontmatter.d.ts +91 -0
  189. package/dist/kb/frontmatter.d.ts.map +1 -0
  190. package/dist/kb/frontmatter.js +274 -0
  191. package/dist/kb/frontmatter.js.map +1 -0
  192. package/dist/kb/line-facts.d.ts +29 -0
  193. package/dist/kb/line-facts.d.ts.map +1 -0
  194. package/dist/kb/line-facts.js +23 -0
  195. package/dist/kb/line-facts.js.map +1 -0
  196. package/dist/kb/page.d.ts +92 -0
  197. package/dist/kb/page.d.ts.map +1 -0
  198. package/dist/kb/page.js +243 -0
  199. package/dist/kb/page.js.map +1 -0
  200. package/dist/kb/scan.d.ts +38 -0
  201. package/dist/kb/scan.d.ts.map +1 -0
  202. package/dist/kb/scan.js +109 -0
  203. package/dist/kb/scan.js.map +1 -0
  204. package/dist/kb/words.d.ts +30 -0
  205. package/dist/kb/words.d.ts.map +1 -0
  206. package/dist/kb/words.js +107 -0
  207. package/dist/kb/words.js.map +1 -0
  208. package/dist/maintenance/adopt.d.ts +65 -0
  209. package/dist/maintenance/adopt.d.ts.map +1 -0
  210. package/dist/maintenance/adopt.js +238 -0
  211. package/dist/maintenance/adopt.js.map +1 -0
  212. package/dist/maintenance/budget.d.ts +55 -0
  213. package/dist/maintenance/budget.d.ts.map +1 -0
  214. package/dist/maintenance/budget.js +61 -0
  215. package/dist/maintenance/budget.js.map +1 -0
  216. package/dist/maintenance/conflicts.d.ts +105 -0
  217. package/dist/maintenance/conflicts.d.ts.map +1 -0
  218. package/dist/maintenance/conflicts.js +368 -0
  219. package/dist/maintenance/conflicts.js.map +1 -0
  220. package/dist/maintenance/contradictions.d.ts +34 -0
  221. package/dist/maintenance/contradictions.d.ts.map +1 -0
  222. package/dist/maintenance/contradictions.js +199 -0
  223. package/dist/maintenance/contradictions.js.map +1 -0
  224. package/dist/maintenance/curate.d.ts +156 -0
  225. package/dist/maintenance/curate.d.ts.map +1 -0
  226. package/dist/maintenance/curate.js +2070 -0
  227. package/dist/maintenance/curate.js.map +1 -0
  228. package/dist/maintenance/dream.d.ts +139 -0
  229. package/dist/maintenance/dream.d.ts.map +1 -0
  230. package/dist/maintenance/dream.js +1632 -0
  231. package/dist/maintenance/dream.js.map +1 -0
  232. package/dist/maintenance/graph-candidates.d.ts +19 -0
  233. package/dist/maintenance/graph-candidates.d.ts.map +1 -0
  234. package/dist/maintenance/graph-candidates.js +120 -0
  235. package/dist/maintenance/graph-candidates.js.map +1 -0
  236. package/dist/maintenance/housekeeping.d.ts +81 -0
  237. package/dist/maintenance/housekeeping.d.ts.map +1 -0
  238. package/dist/maintenance/housekeeping.js +281 -0
  239. package/dist/maintenance/housekeeping.js.map +1 -0
  240. package/dist/maintenance/link-repairs.d.ts +58 -0
  241. package/dist/maintenance/link-repairs.d.ts.map +1 -0
  242. package/dist/maintenance/link-repairs.js +281 -0
  243. package/dist/maintenance/link-repairs.js.map +1 -0
  244. package/dist/maintenance/log.d.ts +41 -0
  245. package/dist/maintenance/log.d.ts.map +1 -0
  246. package/dist/maintenance/log.js +72 -0
  247. package/dist/maintenance/log.js.map +1 -0
  248. package/dist/maintenance/managed-item-routing.d.ts +48 -0
  249. package/dist/maintenance/managed-item-routing.d.ts.map +1 -0
  250. package/dist/maintenance/managed-item-routing.js +281 -0
  251. package/dist/maintenance/managed-item-routing.js.map +1 -0
  252. package/dist/maintenance/managed-item-sources.d.ts +38 -0
  253. package/dist/maintenance/managed-item-sources.d.ts.map +1 -0
  254. package/dist/maintenance/managed-item-sources.js +324 -0
  255. package/dist/maintenance/managed-item-sources.js.map +1 -0
  256. package/dist/maintenance/managed-items.d.ts +162 -0
  257. package/dist/maintenance/managed-items.d.ts.map +1 -0
  258. package/dist/maintenance/managed-items.js +1329 -0
  259. package/dist/maintenance/managed-items.js.map +1 -0
  260. package/dist/maintenance/merge-classifier.d.ts +19 -0
  261. package/dist/maintenance/merge-classifier.d.ts.map +1 -0
  262. package/dist/maintenance/merge-classifier.js +49 -0
  263. package/dist/maintenance/merge-classifier.js.map +1 -0
  264. package/dist/maintenance/model-telemetry.d.ts +45 -0
  265. package/dist/maintenance/model-telemetry.d.ts.map +1 -0
  266. package/dist/maintenance/model-telemetry.js +107 -0
  267. package/dist/maintenance/model-telemetry.js.map +1 -0
  268. package/dist/maintenance/observe.d.ts +72 -0
  269. package/dist/maintenance/observe.d.ts.map +1 -0
  270. package/dist/maintenance/observe.js +265 -0
  271. package/dist/maintenance/observe.js.map +1 -0
  272. package/dist/maintenance/path-policy.d.ts +83 -0
  273. package/dist/maintenance/path-policy.d.ts.map +1 -0
  274. package/dist/maintenance/path-policy.js +360 -0
  275. package/dist/maintenance/path-policy.js.map +1 -0
  276. package/dist/maintenance/plans.d.ts +366 -0
  277. package/dist/maintenance/plans.d.ts.map +1 -0
  278. package/dist/maintenance/plans.js +4060 -0
  279. package/dist/maintenance/plans.js.map +1 -0
  280. package/dist/maintenance/profile.d.ts +33 -0
  281. package/dist/maintenance/profile.d.ts.map +1 -0
  282. package/dist/maintenance/profile.js +99 -0
  283. package/dist/maintenance/profile.js.map +1 -0
  284. package/dist/maintenance/recovery.d.ts +38 -0
  285. package/dist/maintenance/recovery.d.ts.map +1 -0
  286. package/dist/maintenance/recovery.js +217 -0
  287. package/dist/maintenance/recovery.js.map +1 -0
  288. package/dist/maintenance/repair.d.ts +37 -0
  289. package/dist/maintenance/repair.d.ts.map +1 -0
  290. package/dist/maintenance/repair.js +183 -0
  291. package/dist/maintenance/repair.js.map +1 -0
  292. package/dist/maintenance/rule-drift.d.ts +104 -0
  293. package/dist/maintenance/rule-drift.d.ts.map +1 -0
  294. package/dist/maintenance/rule-drift.js +507 -0
  295. package/dist/maintenance/rule-drift.js.map +1 -0
  296. package/dist/maintenance/run-verification.d.ts +44 -0
  297. package/dist/maintenance/run-verification.d.ts.map +1 -0
  298. package/dist/maintenance/run-verification.js +253 -0
  299. package/dist/maintenance/run-verification.js.map +1 -0
  300. package/dist/maintenance/runs.d.ts +142 -0
  301. package/dist/maintenance/runs.d.ts.map +1 -0
  302. package/dist/maintenance/runs.js +439 -0
  303. package/dist/maintenance/runs.js.map +1 -0
  304. package/dist/maintenance/semantic-merge-discovery.d.ts +54 -0
  305. package/dist/maintenance/semantic-merge-discovery.d.ts.map +1 -0
  306. package/dist/maintenance/semantic-merge-discovery.js +308 -0
  307. package/dist/maintenance/semantic-merge-discovery.js.map +1 -0
  308. package/dist/maintenance/temporal.d.ts +44 -0
  309. package/dist/maintenance/temporal.d.ts.map +1 -0
  310. package/dist/maintenance/temporal.js +295 -0
  311. package/dist/maintenance/temporal.js.map +1 -0
  312. package/dist/models/client.d.ts +214 -0
  313. package/dist/models/client.d.ts.map +1 -0
  314. package/dist/models/client.js +904 -0
  315. package/dist/models/client.js.map +1 -0
  316. package/dist/models/provider-api.d.ts +28 -0
  317. package/dist/models/provider-api.d.ts.map +1 -0
  318. package/dist/models/provider-api.js +320 -0
  319. package/dist/models/provider-api.js.map +1 -0
  320. package/dist/open.d.ts +163 -0
  321. package/dist/open.d.ts.map +1 -0
  322. package/dist/open.js +382 -0
  323. package/dist/open.js.map +1 -0
  324. package/dist/ops/adopt.d.ts +12 -0
  325. package/dist/ops/adopt.d.ts.map +1 -0
  326. package/dist/ops/adopt.js +138 -0
  327. package/dist/ops/adopt.js.map +1 -0
  328. package/dist/ops/answer.d.ts +29 -0
  329. package/dist/ops/answer.d.ts.map +1 -0
  330. package/dist/ops/answer.js +557 -0
  331. package/dist/ops/answer.js.map +1 -0
  332. package/dist/ops/context.d.ts +18 -0
  333. package/dist/ops/context.d.ts.map +1 -0
  334. package/dist/ops/context.js +713 -0
  335. package/dist/ops/context.js.map +1 -0
  336. package/dist/ops/folder.d.ts +23 -0
  337. package/dist/ops/folder.d.ts.map +1 -0
  338. package/dist/ops/folder.js +130 -0
  339. package/dist/ops/folder.js.map +1 -0
  340. package/dist/ops/forget.d.ts +14 -0
  341. package/dist/ops/forget.d.ts.map +1 -0
  342. package/dist/ops/forget.js +212 -0
  343. package/dist/ops/forget.js.map +1 -0
  344. package/dist/ops/graph.d.ts +10 -0
  345. package/dist/ops/graph.d.ts.map +1 -0
  346. package/dist/ops/graph.js +480 -0
  347. package/dist/ops/graph.js.map +1 -0
  348. package/dist/ops/ingest.d.ts +40 -0
  349. package/dist/ops/ingest.d.ts.map +1 -0
  350. package/dist/ops/ingest.js +434 -0
  351. package/dist/ops/ingest.js.map +1 -0
  352. package/dist/ops/list.d.ts +9 -0
  353. package/dist/ops/list.d.ts.map +1 -0
  354. package/dist/ops/list.js +217 -0
  355. package/dist/ops/list.js.map +1 -0
  356. package/dist/ops/move.d.ts +20 -0
  357. package/dist/ops/move.d.ts.map +1 -0
  358. package/dist/ops/move.js +173 -0
  359. package/dist/ops/move.js.map +1 -0
  360. package/dist/ops/read.d.ts +9 -0
  361. package/dist/ops/read.d.ts.map +1 -0
  362. package/dist/ops/read.js +271 -0
  363. package/dist/ops/read.js.map +1 -0
  364. package/dist/ops/recall.d.ts +13 -0
  365. package/dist/ops/recall.d.ts.map +1 -0
  366. package/dist/ops/recall.js +297 -0
  367. package/dist/ops/recall.js.map +1 -0
  368. package/dist/ops/remember.d.ts +34 -0
  369. package/dist/ops/remember.d.ts.map +1 -0
  370. package/dist/ops/remember.js +660 -0
  371. package/dist/ops/remember.js.map +1 -0
  372. package/dist/ops/timeline.d.ts +11 -0
  373. package/dist/ops/timeline.d.ts.map +1 -0
  374. package/dist/ops/timeline.js +111 -0
  375. package/dist/ops/timeline.js.map +1 -0
  376. package/dist/ops/undo.d.ts +9 -0
  377. package/dist/ops/undo.d.ts.map +1 -0
  378. package/dist/ops/undo.js +38 -0
  379. package/dist/ops/undo.js.map +1 -0
  380. package/dist/ops/write.d.ts +55 -0
  381. package/dist/ops/write.d.ts.map +1 -0
  382. package/dist/ops/write.js +463 -0
  383. package/dist/ops/write.js.map +1 -0
  384. package/dist/recall/assemble.d.ts +81 -0
  385. package/dist/recall/assemble.d.ts.map +1 -0
  386. package/dist/recall/assemble.js +603 -0
  387. package/dist/recall/assemble.js.map +1 -0
  388. package/dist/recall/expand.d.ts +55 -0
  389. package/dist/recall/expand.d.ts.map +1 -0
  390. package/dist/recall/expand.js +282 -0
  391. package/dist/recall/expand.js.map +1 -0
  392. package/dist/recall/graph-arm.d.ts +17 -0
  393. package/dist/recall/graph-arm.d.ts.map +1 -0
  394. package/dist/recall/graph-arm.js +227 -0
  395. package/dist/recall/graph-arm.js.map +1 -0
  396. package/dist/recall/llm-rerank.d.ts +55 -0
  397. package/dist/recall/llm-rerank.d.ts.map +1 -0
  398. package/dist/recall/llm-rerank.js +213 -0
  399. package/dist/recall/llm-rerank.js.map +1 -0
  400. package/dist/recall/reranker-calibration.d.ts +23 -0
  401. package/dist/recall/reranker-calibration.d.ts.map +1 -0
  402. package/dist/recall/reranker-calibration.js +161 -0
  403. package/dist/recall/reranker-calibration.js.map +1 -0
  404. package/dist/recall/search.d.ts +107 -0
  405. package/dist/recall/search.d.ts.map +1 -0
  406. package/dist/recall/search.js +530 -0
  407. package/dist/recall/search.js.map +1 -0
  408. package/dist/reserved.d.ts +39 -0
  409. package/dist/reserved.d.ts.map +1 -0
  410. package/dist/reserved.js +63 -0
  411. package/dist/reserved.js.map +1 -0
  412. package/dist/rules/compile.d.ts +27 -0
  413. package/dist/rules/compile.d.ts.map +1 -0
  414. package/dist/rules/compile.js +105 -0
  415. package/dist/rules/compile.js.map +1 -0
  416. package/dist/setup/model-free.d.ts +15 -0
  417. package/dist/setup/model-free.d.ts.map +1 -0
  418. package/dist/setup/model-free.js +26 -0
  419. package/dist/setup/model-free.js.map +1 -0
  420. package/dist/setup/openai.d.ts +46 -0
  421. package/dist/setup/openai.d.ts.map +1 -0
  422. package/dist/setup/openai.js +165 -0
  423. package/dist/setup/openai.js.map +1 -0
  424. package/dist/store/db.d.ts +61 -0
  425. package/dist/store/db.d.ts.map +1 -0
  426. package/dist/store/db.js +265 -0
  427. package/dist/store/db.js.map +1 -0
  428. package/dist/store/ids.d.ts +29 -0
  429. package/dist/store/ids.d.ts.map +1 -0
  430. package/dist/store/ids.js +0 -0
  431. package/dist/store/ids.js.map +1 -0
  432. package/dist/store/migrations.d.ts +54 -0
  433. package/dist/store/migrations.d.ts.map +1 -0
  434. package/dist/store/migrations.js +923 -0
  435. package/dist/store/migrations.js.map +1 -0
  436. package/dist/store/vectors.d.ts +34 -0
  437. package/dist/store/vectors.d.ts.map +1 -0
  438. package/dist/store/vectors.js +158 -0
  439. package/dist/store/vectors.js.map +1 -0
  440. package/dist/timeline/documents.d.ts +23 -0
  441. package/dist/timeline/documents.d.ts.map +1 -0
  442. package/dist/timeline/documents.js +262 -0
  443. package/dist/timeline/documents.js.map +1 -0
  444. package/dist/watch/watcher.d.ts +43 -0
  445. package/dist/watch/watcher.d.ts.map +1 -0
  446. package/dist/watch/watcher.js +143 -0
  447. package/dist/watch/watcher.js.map +1 -0
  448. package/dist/write/atomic.d.ts +34 -0
  449. package/dist/write/atomic.d.ts.map +1 -0
  450. package/dist/write/atomic.js +58 -0
  451. package/dist/write/atomic.js.map +1 -0
  452. package/dist/write/conflict.d.ts +43 -0
  453. package/dist/write/conflict.d.ts.map +1 -0
  454. package/dist/write/conflict.js +149 -0
  455. package/dist/write/conflict.js.map +1 -0
  456. package/dist/write/edit.d.ts +59 -0
  457. package/dist/write/edit.d.ts.map +1 -0
  458. package/dist/write/edit.js +231 -0
  459. package/dist/write/edit.js.map +1 -0
  460. package/dist/write/gate.d.ts +59 -0
  461. package/dist/write/gate.d.ts.map +1 -0
  462. package/dist/write/gate.js +99 -0
  463. package/dist/write/gate.js.map +1 -0
  464. package/dist/write/journal.d.ts +90 -0
  465. package/dist/write/journal.d.ts.map +1 -0
  466. package/dist/write/journal.js +187 -0
  467. package/dist/write/journal.js.map +1 -0
  468. package/dist/write/ledger.d.ts +28 -0
  469. package/dist/write/ledger.d.ts.map +1 -0
  470. package/dist/write/ledger.js +141 -0
  471. package/dist/write/ledger.js.map +1 -0
  472. package/dist/write/placement.d.ts +29 -0
  473. package/dist/write/placement.d.ts.map +1 -0
  474. package/dist/write/placement.js +140 -0
  475. package/dist/write/placement.js.map +1 -0
  476. package/dist/write/remember-fallback.d.ts +18 -0
  477. package/dist/write/remember-fallback.d.ts.map +1 -0
  478. package/dist/write/remember-fallback.js +48 -0
  479. package/dist/write/remember-fallback.js.map +1 -0
  480. package/dist/write/retain.d.ts +61 -0
  481. package/dist/write/retain.d.ts.map +1 -0
  482. package/dist/write/retain.js +272 -0
  483. package/dist/write/retain.js.map +1 -0
  484. package/package.json +50 -0
  485. package/swift/extract.swift +232 -0
@@ -0,0 +1 @@
1
+ {"version":3,"file":"remember-fallback.js","sourceRoot":"","sources":["../../src/write/remember-fallback.ts"],"names":[],"mappings":"AAAA,OAAO,GAAG,MAAM,kBAAkB,CAAC;AACnC,OAAO,IAAI,MAAM,WAAW,CAAC;AAE7B,OAAO,EAAE,aAAa,EAA2B,MAAM,kBAAkB,CAAC;AAC1E,OAAO,EAAE,UAAU,EAAE,MAAM,gBAAgB,CAAC;AAS5C;;;GAGG;AACH,MAAM,CAAC,KAAK,UAAU,uBAAuB,CAC3C,GAAgB,EAChB,UAAgC,aAAa,CAAC,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,KAAK,CAAC;IAEpE,MAAM,IAAI,GAAG,GAAG,CAAC,MAAM,CAAC,WAAW,CAAC,MAAM,CAAC,YAAY,CAAC;IACxD,IAAI,CAAC,IAAI;QAAE,OAAO,IAAI,CAAC;IACvB,IAAI,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,MAAM,CAAC;QAAE,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,eAAe,EAAE,CAAC;IAElG,MAAM,GAAG,GAAG,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC,OAAO,CAAC,4DAA4D,CAAC,CAAC,GAAG,CAAC,IAAI,CAC5C,CAAC;IAC5D,IAAI,GAAG,EAAE,CAAC;QACR,OAAO,GAAG,CAAC,IAAI,KAAK,WAAW,IAAI,GAAG,CAAC,mBAAmB,KAAK,WAAW;YACxE,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,eAAe,EAAE;YACnC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,4BAA4B,EAAE,CAAC;IAC5E,CAAC;IAED,0GAA0G;IAC1G,wFAAwF;IACxF,IAAI,MAAM,kBAAkB,CAAC,GAAG,EAAE,IAAI,CAAC,EAAE,CAAC;QACxC,OAAO,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,uBAAuB,EAAE,CAAC;IAC1E,CAAC;IAED,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;IACpD,MAAM,QAAQ,GAAG,OAAO,CAAC,IAAI,CAC3B,CAAC,MAAM,EAAE,EAAE,CACT,MAAM,CAAC,IAAI,KAAK,MAAM;QACtB,MAAM,CAAC,IAAI,KAAK,WAAW;QAC3B,MAAM,CAAC,QAAQ,KAAK,WAAW;QAC/B,MAAM,CAAC,SAAS,CACnB,CAAC;IACF,OAAO,QAAQ;QACb,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,UAAU,EAAE;QAC9B,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,aAAa,EAAE,MAAM,EAAE,qBAAqB,EAAE,CAAC;AACrE,CAAC;AAED,KAAK,UAAU,kBAAkB,CAAC,GAAgB,EAAE,IAAY;IAC9D,KAAK,MAAM,SAAS,IAAI,GAAG,CAAC,MAAM,CAAC,cAAc,EAAE,CAAC;QAClD,IAAI,CAAC;YACH,IAAI,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,GAAG,IAAI,GAAG,SAAS,EAAE,CAAC,CAAC,CAAC,CAAC,MAAM,EAAE;gBAAE,OAAO,IAAI,CAAC;QACpG,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ;gBAAE,MAAM,KAAK,CAAC;QACtE,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}
@@ -0,0 +1,61 @@
1
+ import { z } from 'zod';
2
+ import { type ModelClient } from '../models/client.ts';
3
+ import type { FolderCatalogEntry } from '../kb/folders.ts';
4
+ /**
5
+ * `page` is nullable rather than optional, and the prompt above says "null" rather than "omit"
6
+ * for the same reason: OpenAI's strict mode rejects an optional property outright, and a schema
7
+ * it rejects takes this call site off constrained decoding silently — the fallback to a plain
8
+ * JSON request looks exactly like success. Nullable says the same thing and stays enforceable.
9
+ *
10
+ * The guards this cannot express stay downstream regardless: `cleanCandidates` drops
11
+ * speculation, fragments, and any page whose folder is not one that actually exists.
12
+ */
13
+ export declare const RETAIN_SCHEMA: z.ZodObject<{
14
+ candidates: z.ZodArray<z.ZodObject<{
15
+ text: z.ZodString;
16
+ subject: z.ZodString;
17
+ page: z.ZodNullable<z.ZodString>;
18
+ origin: z.ZodEnum<{
19
+ user: "user";
20
+ assistant: "assistant";
21
+ }>;
22
+ evidence: z.ZodNullable<z.ZodString>;
23
+ kind: z.ZodString;
24
+ }, z.core.$strip>>;
25
+ events: z.ZodArray<z.ZodObject<{
26
+ date: z.ZodString;
27
+ summary: z.ZodString;
28
+ }, z.core.$strip>>;
29
+ }, z.core.$strip>;
30
+ export interface RetainCandidate {
31
+ text: string;
32
+ subject: string;
33
+ kind: string;
34
+ /**
35
+ * Where this would go if no page holds it yet. A suggestion, used only when routing finds
36
+ * nothing — which is the case `remember` used to have no answer for at all.
37
+ */
38
+ page?: string;
39
+ origin?: 'user' | 'assistant';
40
+ /** Exact bounded input bytes retained privately so the managed sentence can be re-verified. */
41
+ evidence?: string;
42
+ }
43
+ export interface RetainResult {
44
+ candidates: RetainCandidate[];
45
+ events: {
46
+ date: string;
47
+ summary: string;
48
+ }[];
49
+ error: string | null;
50
+ }
51
+ export declare function runRetain(text: string, model: ModelClient, options?: {
52
+ mission?: string;
53
+ today: string;
54
+ folders?: FolderCatalogEntry[];
55
+ }): Promise<RetainResult>;
56
+ export declare function cleanCandidates(value: unknown, options?: {
57
+ folders?: readonly string[];
58
+ pages?: readonly string[];
59
+ sourceText?: string;
60
+ }): RetainCandidate[];
61
+ //# sourceMappingURL=retain.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"retain.d.ts","sourceRoot":"","sources":["../../src/write/retain.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,qBAAqB,CAAC;AACvE,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAmE3D;;;;;;;;GAQG;AACH,eAAO,MAAM,aAAa;;;;;;;;;;;;;;;;iBAYxB,CAAC;AAEH,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb;;;OAGG;IACH,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,MAAM,GAAG,WAAW,CAAC;IAC9B,+FAA+F;IAC/F,QAAQ,CAAC,EAAE,MAAM,CAAC;CACnB;AAED,MAAM,WAAW,YAAY;IAC3B,UAAU,EAAE,eAAe,EAAE,CAAC;IAC9B,MAAM,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,OAAO,EAAE,MAAM,CAAA;KAAE,EAAE,CAAC;IAC5C,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAED,wBAAsB,SAAS,CAC7B,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,WAAW,EAClB,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,MAAM,CAAC;IAAC,KAAK,EAAE,MAAM,CAAC;IAAC,OAAO,CAAC,EAAE,kBAAkB,EAAE,CAAA;CAEzE,GACA,OAAO,CAAC,YAAY,CAAC,CAkCvB;AA0DD,wBAAgB,eAAe,CAC7B,KAAK,EAAE,OAAO,EACd,OAAO,GAAE;IAAE,OAAO,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,KAAK,CAAC,EAAE,SAAS,MAAM,EAAE,CAAC;IAAC,UAAU,CAAC,EAAE,MAAM,CAAA;CAAO,GAC5F,eAAe,EAAE,CAsCnB"}
@@ -0,0 +1,272 @@
1
+ import { z } from 'zod';
2
+ import { parseJsonLoose } from "../models/client.js";
3
+ /**
4
+ * **Retain: keep only long-term facts, decisions, preferences, proven
5
+ * experience.**
6
+ *
7
+ * **Retain writes prose, not atomic facts** — it is a
8
+ * page-level curator that edits the right page, and facts are derived beneath it.
9
+ * So a candidate here is a sentence someone would actually write on a page, not a
10
+ * triple.
11
+ *
12
+ * The guardrail about missions applies: a mission string appends emphasis to a
13
+ * fixed system prompt and never replaces it. A replaceable prompt is how every
14
+ * guard gets lost.
15
+ */
16
+ const SYSTEM = `You decide what from a conversation is worth writing into a long-term personal knowledge base, and phrase it as prose.
17
+
18
+ Reply with JSON only:
19
+ {
20
+ "candidates": [
21
+ { "text": "one self-contained sentence, phrased as it should appear on a page",
22
+ "subject": "2-5 words naming what this is about, used to find the right page",
23
+ "page": "folder/page-name",
24
+ "origin": "user or assistant — whose statement established the claim",
25
+ "evidence": "one exact verbatim quote from the input that supports the sentence, or null",
26
+ "kind": "fact" }
27
+ ],
28
+ "events": [ { "date": "YYYY-MM-DD", "summary": "what happened, one clause" } ]
29
+ }
30
+
31
+ Keep:
32
+ - Durable values, dates, identifiers, account details, measurements.
33
+ - Decisions, and the reason for them.
34
+ - Stated preferences and constraints.
35
+ - Proven experience: what was tried, what worked.
36
+ - Findings the assistant established and stated as true: what a thing is, how it works,
37
+ what it costs, which options exist and how they differ. A finding is not a suggestion.
38
+
39
+ Drop, always:
40
+ - Anything true only today: what someone is doing right now, transient state, and any
41
+ reading that expires on its own — prices, weather, availability, rates, live status.
42
+ - Questions, speculation, plans that were not decided.
43
+ - Pleasantries, acknowledgements, and anything the assistant merely proposed or offered.
44
+ - Anything already obviously recorded — do not restate.
45
+ - Anything you inferred rather than were told. Only what the text actually says.
46
+
47
+ Rules:
48
+ - Prose, not triples. "The car insurance premium is now 33 EUR a month" — not "premium=33".
49
+ - Resolve pronouns and relative dates against the text. Never invent a date you were not given.
50
+ - Evidence must be copied byte-for-byte from the input, stay under 1200 characters, and contain
51
+ enough context to verify every durable value in the candidate. Use null when no exact quote does.
52
+ - An "events" entry is something that happened on a date, not a value that is true.
53
+ - "page" is a slug and nothing else: lowercase, hyphenated, no description, no punctuation beyond
54
+ the single "/" between folder and page. Every other field above takes prose, so its example
55
+ doubles as instructions — for this one that is a filename, and a model that copies the wording
56
+ through writes a page named after it. Observed: a whole trip route filed under
57
+ "ada-marlow/projects-taxonomy-branch-and-fallback-page-lowercase-and-hyphenated".
58
+ - Always supply "page" when one of the supplied folders marked eligible=true fits. When the folder
59
+ is creatable, its parent must be that exact folder. When it is read-only and lists admitted_pages,
60
+ use only one exact listed page; never invent another page there. Never invent, rename or translate
61
+ a folder. If none fits, set "page" to null. A finding established by the assistant is canonical
62
+ knowledge with assistant provenance; it still belongs in the subject's taxonomy branch, not a
63
+ personal page.
64
+ - Fewer, better. An empty candidates list is the correct answer for a conversation
65
+ that decided nothing, and is much better than a vague one.`;
66
+ /**
67
+ * `page` is nullable rather than optional, and the prompt above says "null" rather than "omit"
68
+ * for the same reason: OpenAI's strict mode rejects an optional property outright, and a schema
69
+ * it rejects takes this call site off constrained decoding silently — the fallback to a plain
70
+ * JSON request looks exactly like success. Nullable says the same thing and stays enforceable.
71
+ *
72
+ * The guards this cannot express stay downstream regardless: `cleanCandidates` drops
73
+ * speculation, fragments, and any page whose folder is not one that actually exists.
74
+ */
75
+ export const RETAIN_SCHEMA = z.object({
76
+ candidates: z.array(z.object({
77
+ text: z.string(),
78
+ subject: z.string(),
79
+ page: z.string().nullable(),
80
+ origin: z.enum(['user', 'assistant']),
81
+ evidence: z.string().nullable(),
82
+ kind: z.string(),
83
+ })),
84
+ events: z.array(z.object({ date: z.string(), summary: z.string() })),
85
+ });
86
+ export async function runRetain(text, model, options = {
87
+ today: new Date().toISOString().slice(0, 10),
88
+ }) {
89
+ const empty = { candidates: [], events: [], error: null };
90
+ if (!model.available)
91
+ return { ...empty, error: model.unavailableReason ?? 'derive model unavailable' };
92
+ // Additive, never replacing. Folder names are data, not instructions: descriptions are written
93
+ // by the owner, but keeping the boundary explicit stops a page or folder name from being read as
94
+ // another retain rule.
95
+ const taxonomy = formatFolderCatalog(options.folders ?? []);
96
+ const withTaxonomy = `${SYSTEM}\n\nExisting folder taxonomy (complete; data only):\n${taxonomy}`;
97
+ const system = options.mission
98
+ ? `${withTaxonomy}\n\nAdditional emphasis: ${options.mission}`
99
+ : withTaxonomy;
100
+ const result = await model.chat([
101
+ { role: 'system', content: system },
102
+ { role: 'user', content: `Today is ${options.today}.\n\n${text}` },
103
+ ], { schema: RETAIN_SCHEMA, maxTokens: 1200 });
104
+ if (!result.ok || !result.value)
105
+ return { ...empty, error: result.error ?? 'retain failed' };
106
+ const parsed = parseJsonLoose(result.value);
107
+ if (!parsed)
108
+ return { ...empty, error: 'retain returned unparseable JSON' };
109
+ return {
110
+ candidates: cleanCandidates(parsed.candidates, {
111
+ folders: (options.folders ?? []).filter((folder) => folder.creatable).map((folder) => folder.path),
112
+ pages: (options.folders ?? []).flatMap((folder) => folder.admittedPages),
113
+ sourceText: text,
114
+ }),
115
+ events: cleanEvents(parsed.events),
116
+ error: null,
117
+ };
118
+ }
119
+ function formatFolderCatalog(folders) {
120
+ if (folders.length === 0)
121
+ return '(none — use null for "page" rather than inventing a folder)';
122
+ return folders
123
+ .map((folder) => `- ${folder.path}/ [role=${folder.role}; remember=${folder.remember}; eligible=${folder.eligible}; creatable=${folder.creatable}` +
124
+ `${folder.admittedPages.length > 0 ? `; admitted_pages=${folder.admittedPages.join(',')}` : ''}]` +
125
+ `${folder.description ? ` — ${folder.description}` : ''}`)
126
+ .join('\n');
127
+ }
128
+ /**
129
+ * The mission says to drop speculation, and a 3B model does not reliably obey a
130
+ * prompt that says so — observed keeping "travel insurance should be considered"
131
+ * from a source that read "I should probably look into travel insurance at some
132
+ * point". A prompt is a suggestion; this is the layer, which is the whole argument
133
+ * of putting the logic in the layer.
134
+ *
135
+ * Mirrors the hedge list the fact deriver scores on, applied to the candidate
136
+ * itself. A dropped candidate costs nothing — `remember` is not the only way to
137
+ * write — while a speculation written as prose is later recalled *as truth*.
138
+ */
139
+ const SPECULATIVE = /\b(should be considered|should probably|might want|could be worth|worth considering|at some point|look into|maybe|perhaps|probably|possibly|considering whether|thinking about|not sure|tbd|to be decided)\b/i;
140
+ /**
141
+ * A candidate has to read as a **statement, not a topic** — "hotel bill" is not something a
142
+ * page can say.
143
+ *
144
+ * This was a whitelist of forty verbs, and a whitelist is the wrong shape for this test. Every
145
+ * sentence built on a verb nobody thought of was dropped in silence: "the Vision framework
146
+ * OCRs a page in about 0.6 seconds" states a fact and contains no word on that list. The
147
+ * failure fell hardest on exactly the material this pass is worst at keeping — findings about
148
+ * the world, which are phrased with the vocabulary of whatever they are about, while a claim
149
+ * about a household reuses the same dozen verbs forever.
150
+ *
151
+ * So the test is structural instead. A statement has a subject and something predicated of it:
152
+ * at least two words before a verb-shaped token, and a verb-shaped token that is not the first
153
+ * word (an imperative is an instruction, not a claim). Copulas and auxiliaries are still named
154
+ * because they are irregular and no suffix rule reaches them.
155
+ */
156
+ const COPULA = /\b(is|are|was|were|has|have|had|will|would|does|do|did|can|may|must|should)\b/i;
157
+ const VERB_SHAPED = /\b\w{3,}(?:s|ed|es)\b/i;
158
+ function readsAsStatement(text) {
159
+ const words = text.split(/\s+/);
160
+ if (words.length < 4)
161
+ return false;
162
+ if (COPULA.test(text))
163
+ return true;
164
+ // Past the first word: "Book the hotel" is an instruction, and an instruction on a page is
165
+ // read back later as something the household decided.
166
+ const rest = words.slice(1).join(' ');
167
+ return VERB_SHAPED.test(rest);
168
+ }
169
+ export function cleanCandidates(value, options = {}) {
170
+ if (!Array.isArray(value))
171
+ return [];
172
+ const out = [];
173
+ const seen = new Set();
174
+ for (const entry of value) {
175
+ if (typeof entry !== 'object' || entry === null)
176
+ continue;
177
+ const record = entry;
178
+ const text = typeof record.text === 'string' ? record.text.trim().replace(/\s+/g, ' ') : '';
179
+ // A one-word "candidate" is the same fragment problem the fact deriver has:
180
+ // it cannot be read on the page it lands on.
181
+ if (text.split(/\s+/).length < 4 || text.length > 400)
182
+ continue;
183
+ // Speculation, however confidently the model phrased it.
184
+ if (SPECULATIVE.test(text))
185
+ continue;
186
+ if (!readsAsStatement(text))
187
+ continue;
188
+ const key = text.toLowerCase();
189
+ if (seen.has(key))
190
+ continue;
191
+ seen.add(key);
192
+ const cleanedPage = typeof record.page === 'string' ? cleanSlug(record.page) : null;
193
+ const page = cleanedPage && pageIsAdmitted(cleanedPage, options.folders, options.pages) ? cleanedPage : null;
194
+ const evidence = exactCandidateEvidence(record.evidence, options.sourceText);
195
+ out.push({
196
+ text,
197
+ subject: typeof record.subject === 'string' && record.subject.trim().length > 0
198
+ ? record.subject.trim()
199
+ : text.slice(0, 60),
200
+ kind: typeof record.kind === 'string' ? record.kind : 'fact',
201
+ ...(record.origin === 'user' || record.origin === 'assistant' ? { origin: record.origin } : {}),
202
+ ...(page ? { page } : {}),
203
+ ...(evidence ? { evidence } : {}),
204
+ });
205
+ if (out.length >= 12)
206
+ break;
207
+ }
208
+ return out;
209
+ }
210
+ function exactCandidateEvidence(value, sourceText) {
211
+ if (typeof value !== 'string' || sourceText === undefined)
212
+ return null;
213
+ const evidence = value.trim();
214
+ if (evidence.length === 0 || evidence.length > 1200 || evidence.includes('\0'))
215
+ return null;
216
+ return sourceText.includes(evidence) ? evidence : null;
217
+ }
218
+ /** The immediate parent is the filing decision; every deeper segment would be an invented folder. */
219
+ function pageIsAdmitted(page, folders, pages) {
220
+ if (folders === undefined && pages === undefined)
221
+ return true;
222
+ if (pages?.includes(page))
223
+ return true;
224
+ const parent = page.slice(0, page.lastIndexOf('/'));
225
+ return folders?.includes(parent) ?? false;
226
+ }
227
+ /**
228
+ * A model's idea of a slug, made safe to hand to `write`.
229
+ *
230
+ * Deliberately lossy rather than strict: a suggestion that cannot be cleaned into a usable
231
+ * slug is dropped, and routing falls back to asking. A slug is a filename, and the one thing
232
+ * that must not happen is a model naming a path.
233
+ */
234
+ function cleanSlug(raw) {
235
+ const slug = raw
236
+ .trim()
237
+ .toLowerCase()
238
+ .replace(/\.(md|markdown)$/i, '')
239
+ .replace(/[^a-z0-9/\-_ ]+/g, '')
240
+ .replace(/\s+/g, '-')
241
+ .replace(/-{2,}/g, '-')
242
+ .split('/')
243
+ .map((segment) => segment.replace(/^-+|-+$/g, ''))
244
+ .filter((segment) => segment.length > 0 && segment !== '.' && segment !== '..')
245
+ .join('/');
246
+ // One segment is a page at the root of the knowledge base, which is almost never what was
247
+ // meant and is the hardest kind of mess to tidy up later.
248
+ return slug.includes('/') && slug.length <= 120 ? slug : null;
249
+ }
250
+ function cleanEvents(value) {
251
+ if (!Array.isArray(value))
252
+ return [];
253
+ const out = [];
254
+ for (const entry of value) {
255
+ if (typeof entry !== 'object' || entry === null)
256
+ continue;
257
+ const record = entry;
258
+ const date = typeof record.date === 'string' ? record.date.trim() : '';
259
+ const summary = typeof record.summary === 'string' ? record.summary.trim().replace(/\s+/g, ' ') : '';
260
+ // An event with an invented or malformed date is worse than no event: it will
261
+ // sort into the ledger somewhere nobody expects.
262
+ if (!/^\d{4}-\d{2}-\d{2}$/.test(date))
263
+ continue;
264
+ if (summary.length < 4 || summary.length > 300)
265
+ continue;
266
+ out.push({ date, summary });
267
+ if (out.length >= 8)
268
+ break;
269
+ }
270
+ return out;
271
+ }
272
+ //# sourceMappingURL=retain.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"retain.js","sourceRoot":"","sources":["../../src/write/retain.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,cAAc,EAAoB,MAAM,qBAAqB,CAAC;AAGvE;;;;;;;;;;;;GAYG;AAEH,MAAM,MAAM,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;6DAiD8C,CAAC;AAE9D;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,aAAa,GAAG,CAAC,CAAC,MAAM,CAAC;IACpC,UAAU,EAAE,CAAC,CAAC,KAAK,CACjB,CAAC,CAAC,MAAM,CAAC;QACP,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;QAChB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;QACnB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC3B,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;QACrC,QAAQ,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;QAC/B,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;KACjB,CAAC,CACH;IACD,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,CAAC;CACrE,CAAC,CAAC;AAsBH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,IAAY,EACZ,KAAkB,EAClB,UAA+E;IAC7E,KAAK,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;CAC7C;IAED,MAAM,KAAK,GAAiB,EAAE,UAAU,EAAE,EAAE,EAAE,MAAM,EAAE,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC;IACxE,IAAI,CAAC,KAAK,CAAC,SAAS;QAAE,OAAO,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,iBAAiB,IAAI,0BAA0B,EAAE,CAAC;IAExG,+FAA+F;IAC/F,iGAAiG;IACjG,uBAAuB;IACvB,MAAM,QAAQ,GAAG,mBAAmB,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IAC5D,MAAM,YAAY,GAAG,GAAG,MAAM,wDAAwD,QAAQ,EAAE,CAAC;IACjG,MAAM,MAAM,GAAG,OAAO,CAAC,OAAO;QAC5B,CAAC,CAAC,GAAG,YAAY,4BAA4B,OAAO,CAAC,OAAO,EAAE;QAC9D,CAAC,CAAC,YAAY,CAAC;IAEjB,MAAM,MAAM,GAAG,MAAM,KAAK,CAAC,IAAI,CAC7B;QACE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE;QACnC,EAAE,IAAI,EAAE,MAAM,EAAE,OAAO,EAAE,YAAY,OAAO,CAAC,KAAK,QAAQ,IAAI,EAAE,EAAE;KACnE,EACD,EAAE,MAAM,EAAE,aAAa,EAAE,SAAS,EAAE,IAAI,EAAE,CAC3C,CAAC;IACF,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;QAAE,OAAO,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC;IAE7F,MAAM,MAAM,GAAG,cAAc,CAA6C,MAAM,CAAC,KAAK,CAAC,CAAC;IACxF,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,kCAAkC,EAAE,CAAC;IAE5E,OAAO;QACL,UAAU,EAAE,eAAe,CAAC,MAAM,CAAC,UAAU,EAAE;YAC7C,OAAO,EAAE,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,SAAS,CAAC,CAAC,GAAG,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC;YAClG,KAAK,EAAE,CAAC,OAAO,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,CAAC,aAAa,CAAC;YACxE,UAAU,EAAE,IAAI;SACjB,CAAC;QACF,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,MAAM,CAAC;QAClC,KAAK,EAAE,IAAI;KACZ,CAAC;AACJ,CAAC;AAED,SAAS,mBAAmB,CAAC,OAA6B;IACxD,IAAI,OAAO,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,6DAA6D,CAAC;IAC/F,OAAO,OAAO;SACX,GAAG,CACF,CAAC,MAAM,EAAE,EAAE,CACT,KAAK,MAAM,CAAC,IAAI,WAAW,MAAM,CAAC,IAAI,cAAc,MAAM,CAAC,QAAQ,cAAc,MAAM,CAAC,QAAQ,eAAe,MAAM,CAAC,SAAS,EAAE;QACjI,GAAG,MAAM,CAAC,aAAa,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,oBAAoB,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,GAAG;QACjG,GAAG,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,MAAM,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,EAAE,CAC5D;SACA,IAAI,CAAC,IAAI,CAAC,CAAC;AAChB,CAAC;AAED;;;;;;;;;;GAUG;AACH,MAAM,WAAW,GACf,+MAA+M,CAAC;AAElN;;;;;;;;;;;;;;;GAeG;AACH,MAAM,MAAM,GAAG,gFAAgF,CAAC;AAChG,MAAM,WAAW,GAAG,wBAAwB,CAAC;AAE7C,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAChC,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IACnC,IAAI,MAAM,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnC,2FAA2F;IAC3F,sDAAsD;IACtD,MAAM,IAAI,GAAG,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACtC,OAAO,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAChC,CAAC;AAED,MAAM,UAAU,eAAe,CAC7B,KAAc,EACd,UAA2F,EAAE;IAE7F,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,MAAM,GAAG,GAAsB,EAAE,CAAC;IAClC,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAE/B,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;YAAE,SAAS;QAC1D,MAAM,MAAM,GAAG,KAAgC,CAAC;QAChD,MAAM,IAAI,GAAG,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QAC5F,4EAA4E;QAC5E,6CAA6C;QAC7C,IAAI,IAAI,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,GAAG,GAAG;YAAE,SAAS;QAChE,yDAAyD;QACzD,IAAI,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QACrC,IAAI,CAAC,gBAAgB,CAAC,IAAI,CAAC;YAAE,SAAS;QAEtC,MAAM,GAAG,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,SAAS;QAC5B,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;QAEd,MAAM,WAAW,GAAG,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QACpF,MAAM,IAAI,GACR,WAAW,IAAI,cAAc,CAAC,WAAW,EAAE,OAAO,CAAC,OAAO,EAAE,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC;QAClG,MAAM,QAAQ,GAAG,sBAAsB,CAAC,MAAM,CAAC,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC;QAC7E,GAAG,CAAC,IAAI,CAAC;YACP,IAAI;YACJ,OAAO,EACL,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,IAAI,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC;gBACpE,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE;gBACvB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;YACvB,IAAI,EAAE,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM;YAC5D,GAAG,CAAC,MAAM,CAAC,MAAM,KAAK,MAAM,IAAI,MAAM,CAAC,MAAM,KAAK,WAAW,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YAC/F,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;YACzB,GAAG,CAAC,QAAQ,CAAC,CAAC,CAAC,EAAE,QAAQ,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;SAClC,CAAC,CAAC;QACH,IAAI,GAAG,CAAC,MAAM,IAAI,EAAE;YAAE,MAAM;IAC9B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,sBAAsB,CAAC,KAAc,EAAE,UAA8B;IAC5E,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,UAAU,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IACvE,MAAM,QAAQ,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC;IAC9B,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,IAAI,QAAQ,CAAC,MAAM,GAAG,IAAI,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAC5F,OAAO,UAAU,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,IAAI,CAAC;AACzD,CAAC;AAED,qGAAqG;AACrG,SAAS,cAAc,CACrB,IAAY,EACZ,OAAsC,EACtC,KAAoC;IAEpC,IAAI,OAAO,KAAK,SAAS,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,IAAI,CAAC;IAC9D,IAAI,KAAK,EAAE,QAAQ,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IACvC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,CAAC;IACpD,OAAO,OAAO,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,KAAK,CAAC;AAC5C,CAAC;AAED;;;;;;GAMG;AACH,SAAS,SAAS,CAAC,GAAW;IAC5B,MAAM,IAAI,GAAG,GAAG;SACb,IAAI,EAAE;SACN,WAAW,EAAE;SACb,OAAO,CAAC,mBAAmB,EAAE,EAAE,CAAC;SAChC,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC/B,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC;SACpB,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC;SACtB,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC;SACjD,MAAM,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,KAAK,GAAG,IAAI,OAAO,KAAK,IAAI,CAAC;SAC9E,IAAI,CAAC,GAAG,CAAC,CAAC;IAEb,0FAA0F;IAC1F,0DAA0D;IAC1D,OAAO,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,MAAM,IAAI,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AAChE,CAAC;AAED,SAAS,WAAW,CAAC,KAAc;IACjC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,EAAE,CAAC;IACrC,MAAM,GAAG,GAAwC,EAAE,CAAC;IACpD,KAAK,MAAM,KAAK,IAAI,KAAK,EAAE,CAAC;QAC1B,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI;YAAE,SAAS;QAC1D,MAAM,MAAM,GAAG,KAAgC,CAAC;QAChD,MAAM,IAAI,GAAG,OAAO,MAAM,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,MAAM,OAAO,GAAG,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACrG,8EAA8E;QAC9E,iDAAiD;QACjD,IAAI,CAAC,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC;YAAE,SAAS;QAChD,IAAI,OAAO,CAAC,MAAM,GAAG,CAAC,IAAI,OAAO,CAAC,MAAM,GAAG,GAAG;YAAE,SAAS;QACzD,GAAG,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC;QAC5B,IAAI,GAAG,CAAC,MAAM,IAAI,CAAC;YAAE,MAAM;IAC7B,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC"}
package/package.json ADDED
@@ -0,0 +1,50 @@
1
+ {
2
+ "name": "@tenphi/akno-core",
3
+ "version": "0.1.0",
4
+ "description": "Akno — the memory layer itself: index, ops, watcher, models",
5
+ "license": "PolyForm-Noncommercial-1.0.0",
6
+ "type": "module",
7
+ "main": "./dist/index.js",
8
+ "types": "./dist/index.d.ts",
9
+ "exports": {
10
+ ".": {
11
+ "types": "./dist/index.d.ts",
12
+ "default": "./dist/index.js"
13
+ }
14
+ },
15
+ "files": [
16
+ "dist",
17
+ "config",
18
+ "swift"
19
+ ],
20
+ "dependencies": {
21
+ "better-sqlite3": "^12.4.1",
22
+ "sqlite-vec": "^0.1.9",
23
+ "yaml": "^2.9.0",
24
+ "zod": "^4.4.3",
25
+ "@tenphi/akno-protocol": "0.1.0"
26
+ },
27
+ "devDependencies": {
28
+ "@types/better-sqlite3": "^7.6.12"
29
+ },
30
+ "os": [
31
+ "darwin"
32
+ ],
33
+ "engines": {
34
+ "node": ">=22.18"
35
+ },
36
+ "publishConfig": {
37
+ "access": "public"
38
+ },
39
+ "repository": {
40
+ "type": "git",
41
+ "url": "git+https://github.com/tenphi/akno.git",
42
+ "directory": "packages/core"
43
+ },
44
+ "homepage": "https://github.com/tenphi/akno#readme",
45
+ "bugs": {
46
+ "url": "https://github.com/tenphi/akno/issues"
47
+ },
48
+ "author": "Andrey Yamanov",
49
+ "scripts": {}
50
+ }
@@ -0,0 +1,232 @@
1
+ // Akno's text extractor, built on what macOS already has.
2
+ //
3
+ // Extraction happens on arrival, always — text layer first, OCR for scans. The
4
+ // usual way to get that is `brew install poppler tesseract`, which makes a memory
5
+ // layer's first-run experience depend on two unrelated projects being installed and
6
+ // on their CLI flags not changing.
7
+ //
8
+ // macOS ships both capabilities: PDFKit reads a text layer, and Vision does OCR that
9
+ // is faster and more accurate than tesseract, hardware-accelerated, and offline.
10
+ // Since Akno is macOS-only on purpose (see README), using the platform's own
11
+ // frameworks is the honest choice rather than a shortcut.
12
+ //
13
+ // Compiled on first use and cached in the state directory. Output is JSON on stdout
14
+ // so the caller never parses prose.
15
+
16
+ import Foundation
17
+ import PDFKit
18
+ import Vision
19
+ import CoreGraphics
20
+ import ImageIO
21
+
22
+ /// One page's worth of text, so a citation can name the page it came from.
23
+ ///
24
+ /// A card points at the page, the document, **and the page number within it**.
25
+ /// One joined blob cannot support that last part, and a quote from page 9 of a contract
26
+ /// attributed to no page is a citation a reader cannot check.
27
+ struct Section: Encodable {
28
+ var page: Int
29
+ var text: String
30
+ }
31
+
32
+ struct Result: Encodable {
33
+ var text: String
34
+ var pages: Int
35
+ var ocr: Bool
36
+ /// Vision's own mean confidence over recognised lines, 0..1. Absent for a text
37
+ /// layer, where there is nothing to be unsure about.
38
+ var confidence: Double?
39
+ var sections: [Section]?
40
+ var error: String?
41
+ }
42
+
43
+ func emit(_ result: Result) -> Never {
44
+ let encoder = JSONEncoder()
45
+ encoder.outputFormatting = []
46
+ if let data = try? encoder.encode(result), let json = String(data: data, encoding: .utf8) {
47
+ print(json)
48
+ } else {
49
+ print("{\"text\":\"\",\"pages\":0,\"ocr\":false,\"error\":\"could not encode result\"}")
50
+ }
51
+ exit(0)
52
+ }
53
+
54
+ func fail(_ message: String) -> Never {
55
+ emit(Result(text: "", pages: 0, ocr: false, confidence: nil, sections: nil, error: message))
56
+ }
57
+
58
+ /// Vision text recognition over one image. `accurate` rather than `fast`: this runs
59
+ /// once per document ever, and a wrong character in a policy number is permanent.
60
+ func recognise(_ image: CGImage) -> (String, Double)? {
61
+ let request = VNRecognizeTextRequest()
62
+ request.recognitionLevel = .accurate
63
+ request.usesLanguageCorrection = true
64
+ // Documents in this knowledge base are mixed-language; leaving the list broad
65
+ // costs nothing measurable and stops a Dutch contract being read as English.
66
+ if #available(macOS 13.0, *) {
67
+ request.revision = VNRecognizeTextRequestRevision3
68
+ }
69
+ request.recognitionLanguages = ["en-US", "nl-NL", "de-DE", "fr-FR", "es-ES", "ru-RU"]
70
+
71
+ let handler = VNImageRequestHandler(cgImage: image, options: [:])
72
+ do {
73
+ try handler.perform([request])
74
+ } catch {
75
+ return nil
76
+ }
77
+
78
+ guard let observations = request.results, !observations.isEmpty else { return ("", 0) }
79
+
80
+ var lines: [String] = []
81
+ var total = 0.0
82
+ for observation in observations {
83
+ guard let candidate = observation.topCandidates(1).first else { continue }
84
+ lines.append(candidate.string)
85
+ total += Double(candidate.confidence)
86
+ }
87
+ let mean = observations.isEmpty ? 0 : total / Double(observations.count)
88
+ return (lines.joined(separator: "\n"), mean)
89
+ }
90
+
91
+ func loadImage(_ path: String) -> CGImage? {
92
+ guard let source = CGImageSourceCreateWithURL(URL(fileURLWithPath: path) as CFURL, nil) else {
93
+ return nil
94
+ }
95
+ return CGImageSourceCreateImageAtIndex(source, 0, nil)
96
+ }
97
+
98
+ /// Renders a PDF page for OCR. 2x is the sweet spot: enough resolution for Vision to
99
+ /// read small print, without the memory cost of rendering a 300dpi bitmap per page.
100
+ func render(_ page: PDFPage, scale: CGFloat = 2.0) -> CGImage? {
101
+ let bounds = page.bounds(for: .mediaBox)
102
+ let width = Int(bounds.width * scale)
103
+ let height = Int(bounds.height * scale)
104
+ guard width > 0, height > 0, width * height < 40_000_000 else { return nil }
105
+
106
+ guard
107
+ let context = CGContext(
108
+ data: nil,
109
+ width: width,
110
+ height: height,
111
+ bitsPerComponent: 8,
112
+ bytesPerRow: 0,
113
+ space: CGColorSpaceCreateDeviceRGB(),
114
+ bitmapInfo: CGImageAlphaInfo.noneSkipLast.rawValue
115
+ )
116
+ else { return nil }
117
+
118
+ context.setFillColor(CGColor(red: 1, green: 1, blue: 1, alpha: 1))
119
+ context.fill(CGRect(x: 0, y: 0, width: width, height: height))
120
+ context.scaleBy(x: scale, y: scale)
121
+ page.draw(with: .mediaBox, to: context)
122
+ return context.makeImage()
123
+ }
124
+
125
+ // ─── Entry ──────────────────────────────────────────────────────────────────
126
+
127
+ let arguments = CommandLine.arguments
128
+ guard arguments.count >= 3 else {
129
+ fail("usage: akno-extract <pdf|image> <path> [--ocr-pages N] [--force-ocr]")
130
+ }
131
+
132
+ let mode = arguments[1]
133
+ let path = arguments[2]
134
+ var maxOcrPages = 12
135
+ var forceOcr = false
136
+ var index = 3
137
+ while index < arguments.count {
138
+ if arguments[index] == "--ocr-pages", index + 1 < arguments.count {
139
+ maxOcrPages = Int(arguments[index + 1]) ?? maxOcrPages
140
+ index += 2
141
+ } else if arguments[index] == "--force-ocr" {
142
+ forceOcr = true
143
+ index += 1
144
+ } else {
145
+ index += 1
146
+ }
147
+ }
148
+
149
+ guard FileManager.default.fileExists(atPath: path) else { fail("no such file: \(path)") }
150
+
151
+ switch mode {
152
+ case "image":
153
+ guard let image = loadImage(path) else { fail("could not decode the image") }
154
+ guard let (text, confidence) = recognise(image) else { fail("Vision could not run") }
155
+ emit(
156
+ Result(
157
+ text: text,
158
+ pages: 1,
159
+ ocr: true,
160
+ confidence: confidence,
161
+ sections: [Section(page: 1, text: text)],
162
+ error: nil
163
+ )
164
+ )
165
+
166
+ case "pdf":
167
+ guard let document = PDFDocument(url: URL(fileURLWithPath: path)) else {
168
+ fail("could not open the PDF")
169
+ }
170
+ let pageCount = document.pageCount
171
+
172
+ // Text layer first, and that order is not arbitrary: a real text layer is exact,
173
+ // while OCR of the same page is a guess that happens to be usually right.
174
+ if !forceOcr, let layer = document.string {
175
+ let trimmed = layer.trimmingCharacters(in: .whitespacesAndNewlines)
176
+ // A scanned PDF often carries a few stray characters — a stamp, a page
177
+ // number — which is not a text layer. Judged per page so a 40-page scan
178
+ // with one typed cover sheet still gets OCR'd.
179
+ let perPage = pageCount > 0 ? trimmed.count / pageCount : trimmed.count
180
+ if perPage >= 40 {
181
+ // Page by page as well as joined: the join is what a summary reads, the
182
+ // sections are what a citation points into.
183
+ var sections: [Section] = []
184
+ for pageIndex in 0..<pageCount {
185
+ guard let page = document.page(at: pageIndex), let pageText = page.string else { continue }
186
+ let pageTrimmed = pageText.trimmingCharacters(in: .whitespacesAndNewlines)
187
+ if !pageTrimmed.isEmpty { sections.append(Section(page: pageIndex + 1, text: pageTrimmed)) }
188
+ }
189
+ emit(
190
+ Result(
191
+ text: trimmed,
192
+ pages: pageCount,
193
+ ocr: false,
194
+ confidence: nil,
195
+ sections: sections.isEmpty ? nil : sections,
196
+ error: nil
197
+ )
198
+ )
199
+ }
200
+ }
201
+
202
+ // No usable text layer: it is a scan. OCR a bounded number of pages — a 200-page
203
+ // contract does not need every page to be searchable to be findable, and the
204
+ // caller is told how many were read.
205
+ var sections: [Section] = []
206
+ var confidenceTotal = 0.0
207
+ var read = 0
208
+ for pageIndex in 0..<min(pageCount, maxOcrPages) {
209
+ guard let page = document.page(at: pageIndex), let image = render(page) else { continue }
210
+ guard let (text, confidence) = recognise(image) else { continue }
211
+ if !text.isEmpty {
212
+ sections.append(Section(page: pageIndex + 1, text: text))
213
+ confidenceTotal += confidence
214
+ read += 1
215
+ }
216
+ }
217
+
218
+ let mean = read > 0 ? confidenceTotal / Double(read) : 0
219
+ emit(
220
+ Result(
221
+ text: sections.map(\.text).joined(separator: "\n\n"),
222
+ pages: pageCount,
223
+ ocr: true,
224
+ confidence: mean,
225
+ sections: sections.isEmpty ? nil : sections,
226
+ error: nil
227
+ )
228
+ )
229
+
230
+ default:
231
+ fail("unknown mode: \(mode)")
232
+ }