@tenphi/akno-core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +83 -0
- package/README.md +26 -0
- package/config/default.jsonc +548 -0
- package/dist/bench/answer-corpus.d.ts +48 -0
- package/dist/bench/answer-corpus.d.ts.map +1 -0
- package/dist/bench/answer-corpus.js +491 -0
- package/dist/bench/answer-corpus.js.map +1 -0
- package/dist/bench/answer.d.ts +156 -0
- package/dist/bench/answer.d.ts.map +1 -0
- package/dist/bench/answer.js +608 -0
- package/dist/bench/answer.js.map +1 -0
- package/dist/bench/auto-recall-answer-corpus.d.ts +21 -0
- package/dist/bench/auto-recall-answer-corpus.d.ts.map +1 -0
- package/dist/bench/auto-recall-answer-corpus.js +245 -0
- package/dist/bench/auto-recall-answer-corpus.js.map +1 -0
- package/dist/bench/auto-recall-answer.d.ts +198 -0
- package/dist/bench/auto-recall-answer.d.ts.map +1 -0
- package/dist/bench/auto-recall-answer.js +720 -0
- package/dist/bench/auto-recall-answer.js.map +1 -0
- package/dist/bench/auto-recall-corpus.d.ts +45 -0
- package/dist/bench/auto-recall-corpus.d.ts.map +1 -0
- package/dist/bench/auto-recall-corpus.js +221 -0
- package/dist/bench/auto-recall-corpus.js.map +1 -0
- package/dist/bench/auto-recall.d.ts +171 -0
- package/dist/bench/auto-recall.d.ts.map +1 -0
- package/dist/bench/auto-recall.js +651 -0
- package/dist/bench/auto-recall.js.map +1 -0
- package/dist/bench/entity-resolution.d.ts +37 -0
- package/dist/bench/entity-resolution.d.ts.map +1 -0
- package/dist/bench/entity-resolution.js +121 -0
- package/dist/bench/entity-resolution.js.map +1 -0
- package/dist/bench/graph.d.ts +57 -0
- package/dist/bench/graph.d.ts.map +1 -0
- package/dist/bench/graph.js +573 -0
- package/dist/bench/graph.js.map +1 -0
- package/dist/bench/llm-ranking-probe.d.ts +27 -0
- package/dist/bench/llm-ranking-probe.d.ts.map +1 -0
- package/dist/bench/llm-ranking-probe.js +105 -0
- package/dist/bench/llm-ranking-probe.js.map +1 -0
- package/dist/bench/merge-discovery-corpus.d.ts +26 -0
- package/dist/bench/merge-discovery-corpus.d.ts.map +1 -0
- package/dist/bench/merge-discovery-corpus.js +114 -0
- package/dist/bench/merge-discovery-corpus.js.map +1 -0
- package/dist/bench/merge-discovery-review.d.ts +81 -0
- package/dist/bench/merge-discovery-review.d.ts.map +1 -0
- package/dist/bench/merge-discovery-review.js +149 -0
- package/dist/bench/merge-discovery-review.js.map +1 -0
- package/dist/bench/merge-discovery.d.ts +122 -0
- package/dist/bench/merge-discovery.d.ts.map +1 -0
- package/dist/bench/merge-discovery.js +390 -0
- package/dist/bench/merge-discovery.js.map +1 -0
- package/dist/bench/mixed-retrieval.d.ts +35 -0
- package/dist/bench/mixed-retrieval.d.ts.map +1 -0
- package/dist/bench/mixed-retrieval.js +355 -0
- package/dist/bench/mixed-retrieval.js.map +1 -0
- package/dist/bench/ranking-corpus.d.ts +29 -0
- package/dist/bench/ranking-corpus.d.ts.map +1 -0
- package/dist/bench/ranking-corpus.js +514 -0
- package/dist/bench/ranking-corpus.js.map +1 -0
- package/dist/bench/ranking-end-to-end.d.ts +112 -0
- package/dist/bench/ranking-end-to-end.d.ts.map +1 -0
- package/dist/bench/ranking-end-to-end.js +479 -0
- package/dist/bench/ranking-end-to-end.js.map +1 -0
- package/dist/bench/ranking-latency.d.ts +71 -0
- package/dist/bench/ranking-latency.d.ts.map +1 -0
- package/dist/bench/ranking-latency.js +130 -0
- package/dist/bench/ranking-latency.js.map +1 -0
- package/dist/bench/ranking-matrix.d.ts +141 -0
- package/dist/bench/ranking-matrix.d.ts.map +1 -0
- package/dist/bench/ranking-matrix.js +526 -0
- package/dist/bench/ranking-matrix.js.map +1 -0
- package/dist/bench/ranking-review.d.ts +93 -0
- package/dist/bench/ranking-review.d.ts.map +1 -0
- package/dist/bench/ranking-review.js +249 -0
- package/dist/bench/ranking-review.js.map +1 -0
- package/dist/bench/ranking.d.ts +122 -0
- package/dist/bench/ranking.d.ts.map +1 -0
- package/dist/bench/ranking.js +547 -0
- package/dist/bench/ranking.js.map +1 -0
- package/dist/bench.d.ts +60 -0
- package/dist/bench.d.ts.map +1 -0
- package/dist/bench.js +167 -0
- package/dist/bench.js.map +1 -0
- package/dist/config/jsonc.d.ts +8 -0
- package/dist/config/jsonc.d.ts.map +1 -0
- package/dist/config/jsonc.js +85 -0
- package/dist/config/jsonc.js.map +1 -0
- package/dist/config/load.d.ts +43 -0
- package/dist/config/load.d.ts.map +1 -0
- package/dist/config/load.js +608 -0
- package/dist/config/load.js.map +1 -0
- package/dist/config/paths.d.ts +10 -0
- package/dist/config/paths.d.ts.map +1 -0
- package/dist/config/paths.js +38 -0
- package/dist/config/paths.js.map +1 -0
- package/dist/config/schema.d.ts +698 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +394 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/config/write-rules.d.ts +41 -0
- package/dist/config/write-rules.d.ts.map +1 -0
- package/dist/config/write-rules.js +225 -0
- package/dist/config/write-rules.js.map +1 -0
- package/dist/config/write-setup.d.ts +42 -0
- package/dist/config/write-setup.d.ts.map +1 -0
- package/dist/config/write-setup.js +358 -0
- package/dist/config/write-setup.js.map +1 -0
- package/dist/context.d.ts +64 -0
- package/dist/context.d.ts.map +1 -0
- package/dist/context.js +14 -0
- package/dist/context.js.map +1 -0
- package/dist/doctor.d.ts +115 -0
- package/dist/doctor.d.ts.map +1 -0
- package/dist/doctor.js +388 -0
- package/dist/doctor.js.map +1 -0
- package/dist/index/chunk.d.ts +62 -0
- package/dist/index/chunk.d.ts.map +1 -0
- package/dist/index/chunk.js +307 -0
- package/dist/index/chunk.js.map +1 -0
- package/dist/index/defer.d.ts +32 -0
- package/dist/index/defer.d.ts.map +1 -0
- package/dist/index/defer.js +66 -0
- package/dist/index/defer.js.map +1 -0
- package/dist/index/derive.d.ts +99 -0
- package/dist/index/derive.d.ts.map +1 -0
- package/dist/index/derive.js +378 -0
- package/dist/index/derive.js.map +1 -0
- package/dist/index/entity-resolution.d.ts +68 -0
- package/dist/index/entity-resolution.d.ts.map +1 -0
- package/dist/index/entity-resolution.js +305 -0
- package/dist/index/entity-resolution.js.map +1 -0
- package/dist/index/graph.d.ts +46 -0
- package/dist/index/graph.d.ts.map +1 -0
- package/dist/index/graph.js +618 -0
- package/dist/index/graph.js.map +1 -0
- package/dist/index/indexer.d.ts +301 -0
- package/dist/index/indexer.d.ts.map +1 -0
- package/dist/index/indexer.js +1398 -0
- package/dist/index/indexer.js.map +1 -0
- package/dist/index/revision-barrier.d.ts +30 -0
- package/dist/index/revision-barrier.d.ts.map +1 -0
- package/dist/index/revision-barrier.js +140 -0
- package/dist/index/revision-barrier.js.map +1 -0
- package/dist/index.d.ts +55 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +37 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest/adoption-eligibility.d.ts +4 -0
- package/dist/ingest/adoption-eligibility.d.ts.map +1 -0
- package/dist/ingest/adoption-eligibility.js +18 -0
- package/dist/ingest/adoption-eligibility.js.map +1 -0
- package/dist/ingest/availability.d.ts +15 -0
- package/dist/ingest/availability.d.ts.map +1 -0
- package/dist/ingest/availability.js +37 -0
- package/dist/ingest/availability.js.map +1 -0
- package/dist/ingest/extract.d.ts +58 -0
- package/dist/ingest/extract.d.ts.map +1 -0
- package/dist/ingest/extract.js +255 -0
- package/dist/ingest/extract.js.map +1 -0
- package/dist/ingest/fetch.d.ts +35 -0
- package/dist/ingest/fetch.d.ts.map +1 -0
- package/dist/ingest/fetch.js +142 -0
- package/dist/ingest/fetch.js.map +1 -0
- package/dist/ingest/inbox.d.ts +53 -0
- package/dist/ingest/inbox.d.ts.map +1 -0
- package/dist/ingest/inbox.js +110 -0
- package/dist/ingest/inbox.js.map +1 -0
- package/dist/ingest/name.d.ts +72 -0
- package/dist/ingest/name.d.ts.map +1 -0
- package/dist/ingest/name.js +208 -0
- package/dist/ingest/name.js.map +1 -0
- package/dist/ingest/parts.d.ts +78 -0
- package/dist/ingest/parts.d.ts.map +1 -0
- package/dist/ingest/parts.js +52 -0
- package/dist/ingest/parts.js.map +1 -0
- package/dist/ingest/rendition.d.ts +75 -0
- package/dist/ingest/rendition.d.ts.map +1 -0
- package/dist/ingest/rendition.js +92 -0
- package/dist/ingest/rendition.js.map +1 -0
- package/dist/ingest/store.d.ts +66 -0
- package/dist/ingest/store.d.ts.map +1 -0
- package/dist/ingest/store.js +92 -0
- package/dist/ingest/store.js.map +1 -0
- package/dist/kb/folders.d.ts +37 -0
- package/dist/kb/folders.d.ts.map +1 -0
- package/dist/kb/folders.js +133 -0
- package/dist/kb/folders.js.map +1 -0
- package/dist/kb/frontmatter.d.ts +91 -0
- package/dist/kb/frontmatter.d.ts.map +1 -0
- package/dist/kb/frontmatter.js +274 -0
- package/dist/kb/frontmatter.js.map +1 -0
- package/dist/kb/line-facts.d.ts +29 -0
- package/dist/kb/line-facts.d.ts.map +1 -0
- package/dist/kb/line-facts.js +23 -0
- package/dist/kb/line-facts.js.map +1 -0
- package/dist/kb/page.d.ts +92 -0
- package/dist/kb/page.d.ts.map +1 -0
- package/dist/kb/page.js +243 -0
- package/dist/kb/page.js.map +1 -0
- package/dist/kb/scan.d.ts +38 -0
- package/dist/kb/scan.d.ts.map +1 -0
- package/dist/kb/scan.js +109 -0
- package/dist/kb/scan.js.map +1 -0
- package/dist/kb/words.d.ts +30 -0
- package/dist/kb/words.d.ts.map +1 -0
- package/dist/kb/words.js +107 -0
- package/dist/kb/words.js.map +1 -0
- package/dist/maintenance/adopt.d.ts +65 -0
- package/dist/maintenance/adopt.d.ts.map +1 -0
- package/dist/maintenance/adopt.js +238 -0
- package/dist/maintenance/adopt.js.map +1 -0
- package/dist/maintenance/budget.d.ts +55 -0
- package/dist/maintenance/budget.d.ts.map +1 -0
- package/dist/maintenance/budget.js +61 -0
- package/dist/maintenance/budget.js.map +1 -0
- package/dist/maintenance/conflicts.d.ts +105 -0
- package/dist/maintenance/conflicts.d.ts.map +1 -0
- package/dist/maintenance/conflicts.js +368 -0
- package/dist/maintenance/conflicts.js.map +1 -0
- package/dist/maintenance/contradictions.d.ts +34 -0
- package/dist/maintenance/contradictions.d.ts.map +1 -0
- package/dist/maintenance/contradictions.js +199 -0
- package/dist/maintenance/contradictions.js.map +1 -0
- package/dist/maintenance/curate.d.ts +156 -0
- package/dist/maintenance/curate.d.ts.map +1 -0
- package/dist/maintenance/curate.js +2070 -0
- package/dist/maintenance/curate.js.map +1 -0
- package/dist/maintenance/dream.d.ts +139 -0
- package/dist/maintenance/dream.d.ts.map +1 -0
- package/dist/maintenance/dream.js +1632 -0
- package/dist/maintenance/dream.js.map +1 -0
- package/dist/maintenance/graph-candidates.d.ts +19 -0
- package/dist/maintenance/graph-candidates.d.ts.map +1 -0
- package/dist/maintenance/graph-candidates.js +120 -0
- package/dist/maintenance/graph-candidates.js.map +1 -0
- package/dist/maintenance/housekeeping.d.ts +81 -0
- package/dist/maintenance/housekeeping.d.ts.map +1 -0
- package/dist/maintenance/housekeeping.js +281 -0
- package/dist/maintenance/housekeeping.js.map +1 -0
- package/dist/maintenance/link-repairs.d.ts +58 -0
- package/dist/maintenance/link-repairs.d.ts.map +1 -0
- package/dist/maintenance/link-repairs.js +281 -0
- package/dist/maintenance/link-repairs.js.map +1 -0
- package/dist/maintenance/log.d.ts +41 -0
- package/dist/maintenance/log.d.ts.map +1 -0
- package/dist/maintenance/log.js +72 -0
- package/dist/maintenance/log.js.map +1 -0
- package/dist/maintenance/managed-item-routing.d.ts +48 -0
- package/dist/maintenance/managed-item-routing.d.ts.map +1 -0
- package/dist/maintenance/managed-item-routing.js +281 -0
- package/dist/maintenance/managed-item-routing.js.map +1 -0
- package/dist/maintenance/managed-item-sources.d.ts +38 -0
- package/dist/maintenance/managed-item-sources.d.ts.map +1 -0
- package/dist/maintenance/managed-item-sources.js +324 -0
- package/dist/maintenance/managed-item-sources.js.map +1 -0
- package/dist/maintenance/managed-items.d.ts +162 -0
- package/dist/maintenance/managed-items.d.ts.map +1 -0
- package/dist/maintenance/managed-items.js +1329 -0
- package/dist/maintenance/managed-items.js.map +1 -0
- package/dist/maintenance/merge-classifier.d.ts +19 -0
- package/dist/maintenance/merge-classifier.d.ts.map +1 -0
- package/dist/maintenance/merge-classifier.js +49 -0
- package/dist/maintenance/merge-classifier.js.map +1 -0
- package/dist/maintenance/model-telemetry.d.ts +45 -0
- package/dist/maintenance/model-telemetry.d.ts.map +1 -0
- package/dist/maintenance/model-telemetry.js +107 -0
- package/dist/maintenance/model-telemetry.js.map +1 -0
- package/dist/maintenance/observe.d.ts +72 -0
- package/dist/maintenance/observe.d.ts.map +1 -0
- package/dist/maintenance/observe.js +265 -0
- package/dist/maintenance/observe.js.map +1 -0
- package/dist/maintenance/path-policy.d.ts +83 -0
- package/dist/maintenance/path-policy.d.ts.map +1 -0
- package/dist/maintenance/path-policy.js +360 -0
- package/dist/maintenance/path-policy.js.map +1 -0
- package/dist/maintenance/plans.d.ts +366 -0
- package/dist/maintenance/plans.d.ts.map +1 -0
- package/dist/maintenance/plans.js +4060 -0
- package/dist/maintenance/plans.js.map +1 -0
- package/dist/maintenance/profile.d.ts +33 -0
- package/dist/maintenance/profile.d.ts.map +1 -0
- package/dist/maintenance/profile.js +99 -0
- package/dist/maintenance/profile.js.map +1 -0
- package/dist/maintenance/recovery.d.ts +38 -0
- package/dist/maintenance/recovery.d.ts.map +1 -0
- package/dist/maintenance/recovery.js +217 -0
- package/dist/maintenance/recovery.js.map +1 -0
- package/dist/maintenance/repair.d.ts +37 -0
- package/dist/maintenance/repair.d.ts.map +1 -0
- package/dist/maintenance/repair.js +183 -0
- package/dist/maintenance/repair.js.map +1 -0
- package/dist/maintenance/rule-drift.d.ts +104 -0
- package/dist/maintenance/rule-drift.d.ts.map +1 -0
- package/dist/maintenance/rule-drift.js +507 -0
- package/dist/maintenance/rule-drift.js.map +1 -0
- package/dist/maintenance/run-verification.d.ts +44 -0
- package/dist/maintenance/run-verification.d.ts.map +1 -0
- package/dist/maintenance/run-verification.js +253 -0
- package/dist/maintenance/run-verification.js.map +1 -0
- package/dist/maintenance/runs.d.ts +142 -0
- package/dist/maintenance/runs.d.ts.map +1 -0
- package/dist/maintenance/runs.js +439 -0
- package/dist/maintenance/runs.js.map +1 -0
- package/dist/maintenance/semantic-merge-discovery.d.ts +54 -0
- package/dist/maintenance/semantic-merge-discovery.d.ts.map +1 -0
- package/dist/maintenance/semantic-merge-discovery.js +308 -0
- package/dist/maintenance/semantic-merge-discovery.js.map +1 -0
- package/dist/maintenance/temporal.d.ts +44 -0
- package/dist/maintenance/temporal.d.ts.map +1 -0
- package/dist/maintenance/temporal.js +295 -0
- package/dist/maintenance/temporal.js.map +1 -0
- package/dist/models/client.d.ts +214 -0
- package/dist/models/client.d.ts.map +1 -0
- package/dist/models/client.js +904 -0
- package/dist/models/client.js.map +1 -0
- package/dist/models/provider-api.d.ts +28 -0
- package/dist/models/provider-api.d.ts.map +1 -0
- package/dist/models/provider-api.js +320 -0
- package/dist/models/provider-api.js.map +1 -0
- package/dist/open.d.ts +163 -0
- package/dist/open.d.ts.map +1 -0
- package/dist/open.js +382 -0
- package/dist/open.js.map +1 -0
- package/dist/ops/adopt.d.ts +12 -0
- package/dist/ops/adopt.d.ts.map +1 -0
- package/dist/ops/adopt.js +138 -0
- package/dist/ops/adopt.js.map +1 -0
- package/dist/ops/answer.d.ts +29 -0
- package/dist/ops/answer.d.ts.map +1 -0
- package/dist/ops/answer.js +557 -0
- package/dist/ops/answer.js.map +1 -0
- package/dist/ops/context.d.ts +18 -0
- package/dist/ops/context.d.ts.map +1 -0
- package/dist/ops/context.js +713 -0
- package/dist/ops/context.js.map +1 -0
- package/dist/ops/folder.d.ts +23 -0
- package/dist/ops/folder.d.ts.map +1 -0
- package/dist/ops/folder.js +130 -0
- package/dist/ops/folder.js.map +1 -0
- package/dist/ops/forget.d.ts +14 -0
- package/dist/ops/forget.d.ts.map +1 -0
- package/dist/ops/forget.js +212 -0
- package/dist/ops/forget.js.map +1 -0
- package/dist/ops/graph.d.ts +10 -0
- package/dist/ops/graph.d.ts.map +1 -0
- package/dist/ops/graph.js +480 -0
- package/dist/ops/graph.js.map +1 -0
- package/dist/ops/ingest.d.ts +40 -0
- package/dist/ops/ingest.d.ts.map +1 -0
- package/dist/ops/ingest.js +434 -0
- package/dist/ops/ingest.js.map +1 -0
- package/dist/ops/list.d.ts +9 -0
- package/dist/ops/list.d.ts.map +1 -0
- package/dist/ops/list.js +217 -0
- package/dist/ops/list.js.map +1 -0
- package/dist/ops/move.d.ts +20 -0
- package/dist/ops/move.d.ts.map +1 -0
- package/dist/ops/move.js +173 -0
- package/dist/ops/move.js.map +1 -0
- package/dist/ops/read.d.ts +9 -0
- package/dist/ops/read.d.ts.map +1 -0
- package/dist/ops/read.js +271 -0
- package/dist/ops/read.js.map +1 -0
- package/dist/ops/recall.d.ts +13 -0
- package/dist/ops/recall.d.ts.map +1 -0
- package/dist/ops/recall.js +297 -0
- package/dist/ops/recall.js.map +1 -0
- package/dist/ops/remember.d.ts +34 -0
- package/dist/ops/remember.d.ts.map +1 -0
- package/dist/ops/remember.js +660 -0
- package/dist/ops/remember.js.map +1 -0
- package/dist/ops/timeline.d.ts +11 -0
- package/dist/ops/timeline.d.ts.map +1 -0
- package/dist/ops/timeline.js +111 -0
- package/dist/ops/timeline.js.map +1 -0
- package/dist/ops/undo.d.ts +9 -0
- package/dist/ops/undo.d.ts.map +1 -0
- package/dist/ops/undo.js +38 -0
- package/dist/ops/undo.js.map +1 -0
- package/dist/ops/write.d.ts +55 -0
- package/dist/ops/write.d.ts.map +1 -0
- package/dist/ops/write.js +463 -0
- package/dist/ops/write.js.map +1 -0
- package/dist/recall/assemble.d.ts +81 -0
- package/dist/recall/assemble.d.ts.map +1 -0
- package/dist/recall/assemble.js +603 -0
- package/dist/recall/assemble.js.map +1 -0
- package/dist/recall/expand.d.ts +55 -0
- package/dist/recall/expand.d.ts.map +1 -0
- package/dist/recall/expand.js +282 -0
- package/dist/recall/expand.js.map +1 -0
- package/dist/recall/graph-arm.d.ts +17 -0
- package/dist/recall/graph-arm.d.ts.map +1 -0
- package/dist/recall/graph-arm.js +227 -0
- package/dist/recall/graph-arm.js.map +1 -0
- package/dist/recall/llm-rerank.d.ts +55 -0
- package/dist/recall/llm-rerank.d.ts.map +1 -0
- package/dist/recall/llm-rerank.js +213 -0
- package/dist/recall/llm-rerank.js.map +1 -0
- package/dist/recall/reranker-calibration.d.ts +23 -0
- package/dist/recall/reranker-calibration.d.ts.map +1 -0
- package/dist/recall/reranker-calibration.js +161 -0
- package/dist/recall/reranker-calibration.js.map +1 -0
- package/dist/recall/search.d.ts +107 -0
- package/dist/recall/search.d.ts.map +1 -0
- package/dist/recall/search.js +530 -0
- package/dist/recall/search.js.map +1 -0
- package/dist/reserved.d.ts +39 -0
- package/dist/reserved.d.ts.map +1 -0
- package/dist/reserved.js +63 -0
- package/dist/reserved.js.map +1 -0
- package/dist/rules/compile.d.ts +27 -0
- package/dist/rules/compile.d.ts.map +1 -0
- package/dist/rules/compile.js +105 -0
- package/dist/rules/compile.js.map +1 -0
- package/dist/setup/model-free.d.ts +15 -0
- package/dist/setup/model-free.d.ts.map +1 -0
- package/dist/setup/model-free.js +26 -0
- package/dist/setup/model-free.js.map +1 -0
- package/dist/setup/openai.d.ts +46 -0
- package/dist/setup/openai.d.ts.map +1 -0
- package/dist/setup/openai.js +165 -0
- package/dist/setup/openai.js.map +1 -0
- package/dist/store/db.d.ts +61 -0
- package/dist/store/db.d.ts.map +1 -0
- package/dist/store/db.js +265 -0
- package/dist/store/db.js.map +1 -0
- package/dist/store/ids.d.ts +29 -0
- package/dist/store/ids.d.ts.map +1 -0
- package/dist/store/ids.js +0 -0
- package/dist/store/ids.js.map +1 -0
- package/dist/store/migrations.d.ts +54 -0
- package/dist/store/migrations.d.ts.map +1 -0
- package/dist/store/migrations.js +923 -0
- package/dist/store/migrations.js.map +1 -0
- package/dist/store/vectors.d.ts +34 -0
- package/dist/store/vectors.d.ts.map +1 -0
- package/dist/store/vectors.js +158 -0
- package/dist/store/vectors.js.map +1 -0
- package/dist/timeline/documents.d.ts +23 -0
- package/dist/timeline/documents.d.ts.map +1 -0
- package/dist/timeline/documents.js +262 -0
- package/dist/timeline/documents.js.map +1 -0
- package/dist/watch/watcher.d.ts +43 -0
- package/dist/watch/watcher.d.ts.map +1 -0
- package/dist/watch/watcher.js +143 -0
- package/dist/watch/watcher.js.map +1 -0
- package/dist/write/atomic.d.ts +34 -0
- package/dist/write/atomic.d.ts.map +1 -0
- package/dist/write/atomic.js +58 -0
- package/dist/write/atomic.js.map +1 -0
- package/dist/write/conflict.d.ts +43 -0
- package/dist/write/conflict.d.ts.map +1 -0
- package/dist/write/conflict.js +149 -0
- package/dist/write/conflict.js.map +1 -0
- package/dist/write/edit.d.ts +59 -0
- package/dist/write/edit.d.ts.map +1 -0
- package/dist/write/edit.js +231 -0
- package/dist/write/edit.js.map +1 -0
- package/dist/write/gate.d.ts +59 -0
- package/dist/write/gate.d.ts.map +1 -0
- package/dist/write/gate.js +99 -0
- package/dist/write/gate.js.map +1 -0
- package/dist/write/journal.d.ts +90 -0
- package/dist/write/journal.d.ts.map +1 -0
- package/dist/write/journal.js +187 -0
- package/dist/write/journal.js.map +1 -0
- package/dist/write/ledger.d.ts +28 -0
- package/dist/write/ledger.d.ts.map +1 -0
- package/dist/write/ledger.js +141 -0
- package/dist/write/ledger.js.map +1 -0
- package/dist/write/placement.d.ts +29 -0
- package/dist/write/placement.d.ts.map +1 -0
- package/dist/write/placement.js +140 -0
- package/dist/write/placement.js.map +1 -0
- package/dist/write/remember-fallback.d.ts +18 -0
- package/dist/write/remember-fallback.d.ts.map +1 -0
- package/dist/write/remember-fallback.js +48 -0
- package/dist/write/remember-fallback.js.map +1 -0
- package/dist/write/retain.d.ts +61 -0
- package/dist/write/retain.d.ts.map +1 -0
- package/dist/write/retain.js +272 -0
- package/dist/write/retain.js.map +1 -0
- package/package.json +50 -0
- package/swift/extract.swift +232 -0
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { z } from 'zod';
|
|
2
|
+
import { type ModelClient } from '../models/client.ts';
|
|
3
|
+
/**
|
|
4
|
+
* **Files arrive with useless names.** `IMG_4821.HEIC`. `Scan 2026-08-06 at
|
|
5
|
+
* 14.22.pdf`. `document(3).pdf`. A camera or a scanner named these, and the name says
|
|
6
|
+
* nothing about what is inside — six months later nobody can find them, and neither
|
|
7
|
+
* can search, because the filename is indexed and contributes only noise.
|
|
8
|
+
*
|
|
9
|
+
* Akno already has the text. Once it has that, naming is nearly free: **one call to
|
|
10
|
+
* the derive model over the extracted text returns title, slug, summary, type and
|
|
11
|
+
* a suggested folder together.** Not five calls — the model is reading the document
|
|
12
|
+
* once either way.
|
|
13
|
+
*
|
|
14
|
+
* Two guards, both of which matter more than the naming does:
|
|
15
|
+
*
|
|
16
|
+
* - **A good name is left alone.** `2024-lease-agreement.pdf` carries information and
|
|
17
|
+
* is kept. Renaming only fires when the filename adds nothing the content does not
|
|
18
|
+
* already say.
|
|
19
|
+
* - **Low confidence means no rename.** A photo of a garden or a corrupt scan keeps
|
|
20
|
+
* its name, gets no page, and is flagged in `doctor` rather than given a confident
|
|
21
|
+
* wrong one.
|
|
22
|
+
*/
|
|
23
|
+
export interface NamedDocument {
|
|
24
|
+
title: string;
|
|
25
|
+
/** Basename only. The folder is decided by routing, not here. */
|
|
26
|
+
slug: string;
|
|
27
|
+
summary: string;
|
|
28
|
+
type: string | null;
|
|
29
|
+
/** A folder the model thinks fits. Advisory — routing decides. */
|
|
30
|
+
suggestedFolder: string | null;
|
|
31
|
+
/** The model's own confidence that it understood the document at all, 0..1. */
|
|
32
|
+
confidence: number;
|
|
33
|
+
error: string | null;
|
|
34
|
+
}
|
|
35
|
+
/** `folder` is nullable rather than optional: the prompt offers `null` explicitly as the
|
|
36
|
+
* answer for "none fit", and a nullable field keeps that a stated choice instead of an
|
|
37
|
+
* absence that could equally mean the model forgot. */
|
|
38
|
+
export declare const NAME_SCHEMA: z.ZodObject<{
|
|
39
|
+
title: z.ZodString;
|
|
40
|
+
slug: z.ZodString;
|
|
41
|
+
summary: z.ZodString;
|
|
42
|
+
type: z.ZodString;
|
|
43
|
+
folder: z.ZodNullable<z.ZodString>;
|
|
44
|
+
confidence: z.ZodNumber;
|
|
45
|
+
}, z.core.$strip>;
|
|
46
|
+
export interface NameOptions {
|
|
47
|
+
/** Extracted text. Truncated before the call — a name does not need 40 pages. */
|
|
48
|
+
text: string;
|
|
49
|
+
originalName: string;
|
|
50
|
+
/** Folders that exist, so the suggestion is a real place and not an invention. */
|
|
51
|
+
folders: string[];
|
|
52
|
+
derive: ModelClient;
|
|
53
|
+
}
|
|
54
|
+
export declare function nameDocument(options: NameOptions): Promise<NamedDocument>;
|
|
55
|
+
/**
|
|
56
|
+
* A title a small model produced by echoing the slug — `ip-datagrams-on-avian-carriers` —
|
|
57
|
+
* turned back into words.
|
|
58
|
+
*
|
|
59
|
+
* The prompt asks for "what a person would call this", and models answer that with a slug
|
|
60
|
+
* often enough to matter: the title is what every recall card shows, so it is the one
|
|
61
|
+
* field where a machine-shaped answer is most visible. Only fired when there is no space
|
|
62
|
+
* at all, so a hyphenated title someone meant — `Zephyr QX-100 warranty` — is untouched.
|
|
63
|
+
*/
|
|
64
|
+
export declare function unslug(title: string | null): string | null;
|
|
65
|
+
/**
|
|
66
|
+
* A model asked for a filename will sometimes answer with a path, an extension, or
|
|
67
|
+
* a sentence. Only the basename shape survives — the folder is routing's decision,
|
|
68
|
+
* and a slug that can express `../` is one refactor away from being used as a path.
|
|
69
|
+
*/
|
|
70
|
+
export declare function cleanSlug(value: unknown): string | null;
|
|
71
|
+
export declare function nameIsUseless(originalName: string): boolean;
|
|
72
|
+
//# sourceMappingURL=name.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"name.d.ts","sourceRoot":"","sources":["../../src/ingest/name.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAkB,KAAK,WAAW,EAAE,MAAM,qBAAqB,CAAC;AAEvE;;;;;;;;;;;;;;;;;;;GAmBG;AAEH,MAAM,WAAW,aAAa;IAC5B,KAAK,EAAE,MAAM,CAAC;IACd,iEAAiE;IACjE,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC;IACpB,kEAAkE;IAClE,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,+EAA+E;IAC/E,UAAU,EAAE,MAAM,CAAC;IACnB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;CACtB;AAiCD;;wDAEwD;AACxD,eAAO,MAAM,WAAW;;;;;;;iBAOtB,CAAC;AAEH,MAAM,WAAW,WAAW;IAC1B,iFAAiF;IACjF,IAAI,EAAE,MAAM,CAAC;IACb,YAAY,EAAE,MAAM,CAAC;IACrB,kFAAkF;IAClF,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,MAAM,EAAE,WAAW,CAAC;CACrB;AAED,wBAAsB,YAAY,CAAC,OAAO,EAAE,WAAW,GAAG,OAAO,CAAC,aAAa,CAAC,CAsD/E;AAED;;;;;;;;GAQG;AACH,wBAAgB,MAAM,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,MAAM,GAAG,IAAI,CAI1D;AAcD;;;;GAIG;AACH,wBAAgB,SAAS,CAAC,KAAK,EAAE,OAAO,GAAG,MAAM,GAAG,IAAI,CAevD;AAwCD,wBAAgB,aAAa,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAc3D"}
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
import { z } from 'zod';
|
|
3
|
+
import { parseJsonLoose } from "../models/client.js";
|
|
4
|
+
const SYSTEM = `You are naming a document for a personal knowledge base from its extracted text.
|
|
5
|
+
|
|
6
|
+
Reply with JSON only:
|
|
7
|
+
{
|
|
8
|
+
"title": "what a person would call this, 3-8 words",
|
|
9
|
+
"slug": "kebab-case-file-name, no folder, no extension",
|
|
10
|
+
"summary": "one or two sentences: what it is, and its most load-bearing values",
|
|
11
|
+
"type": "one lowercase word, e.g. invoice receipt contract statement policy letter report photo note",
|
|
12
|
+
"folder": "an existing folder from the list you were given, or null if none fit",
|
|
13
|
+
"confidence": 0.0
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
The naming rule is **kind + subject + date**, from the content:
|
|
17
|
+
invoice-northern-water-2026-07
|
|
18
|
+
contract-apartment-lease-2024-08
|
|
19
|
+
receipt-car-rental-2026-06
|
|
20
|
+
|
|
21
|
+
Rules:
|
|
22
|
+
- Name it from what the text says. Never from the original filename — that is the
|
|
23
|
+
thing being replaced.
|
|
24
|
+
- Include a date in the slug when the document has one, in YYYY-MM or YYYY-MM-DD.
|
|
25
|
+
- No personal names in the slug unless the document is *about* that person
|
|
26
|
+
(an identity document, a contract in their name).
|
|
27
|
+
- "confidence" is how sure you are that you understood what this document is:
|
|
28
|
+
0.9+ the text plainly says what it is
|
|
29
|
+
0.5 you can guess from fragments
|
|
30
|
+
0.2 the text is garbled, or too short to tell
|
|
31
|
+
0.0 you cannot tell at all
|
|
32
|
+
Be honest. A low number is useful; a confident wrong name is not.
|
|
33
|
+
- Reply with confidence 0 rather than inventing a title for text you cannot read.`;
|
|
34
|
+
/** `folder` is nullable rather than optional: the prompt offers `null` explicitly as the
|
|
35
|
+
* answer for "none fit", and a nullable field keeps that a stated choice instead of an
|
|
36
|
+
* absence that could equally mean the model forgot. */
|
|
37
|
+
export const NAME_SCHEMA = z.object({
|
|
38
|
+
title: z.string(),
|
|
39
|
+
slug: z.string(),
|
|
40
|
+
summary: z.string(),
|
|
41
|
+
type: z.string(),
|
|
42
|
+
folder: z.string().nullable(),
|
|
43
|
+
confidence: z.number(),
|
|
44
|
+
});
|
|
45
|
+
export async function nameDocument(options) {
|
|
46
|
+
const empty = {
|
|
47
|
+
title: '',
|
|
48
|
+
slug: '',
|
|
49
|
+
summary: '',
|
|
50
|
+
type: null,
|
|
51
|
+
suggestedFolder: null,
|
|
52
|
+
confidence: 0,
|
|
53
|
+
error: null,
|
|
54
|
+
};
|
|
55
|
+
if (!options.derive.available) {
|
|
56
|
+
return { ...empty, error: options.derive.unavailableReason ?? 'derive model unavailable' };
|
|
57
|
+
}
|
|
58
|
+
if (options.text.trim().length < 20) {
|
|
59
|
+
return { ...empty, error: 'too little text to name the document from' };
|
|
60
|
+
}
|
|
61
|
+
// The opening of a document says what it is. 6000 characters covers a multi-page
|
|
62
|
+
// contract's first pages, and sending 40 pages to choose 5 words is waste.
|
|
63
|
+
const excerpt = options.text.slice(0, 6000);
|
|
64
|
+
const folders = options.folders.slice(0, 60).join(', ');
|
|
65
|
+
const result = await options.derive.chat([
|
|
66
|
+
{ role: 'system', content: SYSTEM },
|
|
67
|
+
{
|
|
68
|
+
role: 'user',
|
|
69
|
+
content: `Existing folders: ${folders || '(none yet)'}\n\nExtracted text:\n\n${excerpt}`,
|
|
70
|
+
},
|
|
71
|
+
], { schema: NAME_SCHEMA, maxTokens: 500 });
|
|
72
|
+
if (!result.ok || !result.value)
|
|
73
|
+
return { ...empty, error: result.error ?? 'naming failed' };
|
|
74
|
+
const parsed = parseJsonLoose(result.value);
|
|
75
|
+
if (!parsed)
|
|
76
|
+
return { ...empty, error: 'naming returned unparseable JSON' };
|
|
77
|
+
const slug = cleanSlug(parsed.slug);
|
|
78
|
+
const title = unslug(cleanText(parsed.title, 120));
|
|
79
|
+
if (!slug || !title)
|
|
80
|
+
return { ...empty, error: 'naming returned no usable slug or title' };
|
|
81
|
+
const suggested = cleanText(parsed.folder, 200);
|
|
82
|
+
return {
|
|
83
|
+
title,
|
|
84
|
+
slug,
|
|
85
|
+
summary: cleanText(parsed.summary, 400) ?? '',
|
|
86
|
+
type: cleanType(parsed.type),
|
|
87
|
+
// Only a folder that actually exists. A model inventing `invoices/` and Akno
|
|
88
|
+
// creating it is exactly the action the gate exists to prevent.
|
|
89
|
+
suggestedFolder: suggested && options.folders.includes(suggested) ? suggested : null,
|
|
90
|
+
confidence: clampConfidence(parsed.confidence),
|
|
91
|
+
error: null,
|
|
92
|
+
};
|
|
93
|
+
}
|
|
94
|
+
/**
|
|
95
|
+
* A title a small model produced by echoing the slug — `ip-datagrams-on-avian-carriers` —
|
|
96
|
+
* turned back into words.
|
|
97
|
+
*
|
|
98
|
+
* The prompt asks for "what a person would call this", and models answer that with a slug
|
|
99
|
+
* often enough to matter: the title is what every recall card shows, so it is the one
|
|
100
|
+
* field where a machine-shaped answer is most visible. Only fired when there is no space
|
|
101
|
+
* at all, so a hyphenated title someone meant — `Zephyr QX-100 warranty` — is untouched.
|
|
102
|
+
*/
|
|
103
|
+
export function unslug(title) {
|
|
104
|
+
if (title === null || /\s/.test(title))
|
|
105
|
+
return title;
|
|
106
|
+
if (!/[-_]/.test(title))
|
|
107
|
+
return title;
|
|
108
|
+
return title.replace(/[-_]+/g, ' ');
|
|
109
|
+
}
|
|
110
|
+
function cleanText(value, max) {
|
|
111
|
+
if (typeof value !== 'string')
|
|
112
|
+
return null;
|
|
113
|
+
const trimmed = value.trim().replace(/\s+/g, ' ');
|
|
114
|
+
return trimmed.length > 0 ? trimmed.slice(0, max) : null;
|
|
115
|
+
}
|
|
116
|
+
function cleanType(value) {
|
|
117
|
+
if (typeof value !== 'string')
|
|
118
|
+
return null;
|
|
119
|
+
const type = value.trim().toLowerCase().split(/\s+/)[0] ?? '';
|
|
120
|
+
return /^[a-z][a-z-]{1,30}$/.test(type) ? type : null;
|
|
121
|
+
}
|
|
122
|
+
/**
|
|
123
|
+
* A model asked for a filename will sometimes answer with a path, an extension, or
|
|
124
|
+
* a sentence. Only the basename shape survives — the folder is routing's decision,
|
|
125
|
+
* and a slug that can express `../` is one refactor away from being used as a path.
|
|
126
|
+
*/
|
|
127
|
+
export function cleanSlug(value) {
|
|
128
|
+
if (typeof value !== 'string')
|
|
129
|
+
return null;
|
|
130
|
+
const base = value
|
|
131
|
+
.trim()
|
|
132
|
+
.toLowerCase()
|
|
133
|
+
.split('/')
|
|
134
|
+
.pop()
|
|
135
|
+
.replace(/\.[a-z0-9]{1,8}$/, '')
|
|
136
|
+
.normalize('NFKD')
|
|
137
|
+
.replace(/[\u0300-\u036f]/g, '')
|
|
138
|
+
.replace(/[^a-z0-9]+/g, '-')
|
|
139
|
+
.replace(/^-+|-+$/g, '')
|
|
140
|
+
.slice(0, 80)
|
|
141
|
+
.replace(/-+$/, '');
|
|
142
|
+
return base.length >= 3 ? base : null;
|
|
143
|
+
}
|
|
144
|
+
/**
|
|
145
|
+
* Renaming only fires when the filename adds nothing the content does not
|
|
146
|
+
* already say — camera and scanner patterns, bare hashes and UUIDs, `document(3)`, or
|
|
147
|
+
* a name with no words in it beyond a date.
|
|
148
|
+
*
|
|
149
|
+
* The bias is deliberately toward keeping: a name someone chose carries intent that
|
|
150
|
+
* no model can reconstruct, and getting this wrong is destructive in a way a bad
|
|
151
|
+
* summary is not.
|
|
152
|
+
*/
|
|
153
|
+
const USELESS_NAME = [
|
|
154
|
+
/^img[_-]?\d+$/i,
|
|
155
|
+
/^dsc[_-]?\d+$/i,
|
|
156
|
+
/^dscn\d+$/i,
|
|
157
|
+
/^p\d{7,}$/i,
|
|
158
|
+
// Pixel and other Android cameras: PXL_20260806_141500, MVIMG_, PANO_, VID_.
|
|
159
|
+
/^(pxl|mvimg|pano|vid|burst)[_-]?\d+/i,
|
|
160
|
+
// Photo Booth and the iOS simulator.
|
|
161
|
+
/^photo on \d{4}-\d{2}-\d{2}/i,
|
|
162
|
+
/^simulator screenshot/i,
|
|
163
|
+
/^photo[_ -]?\d*$/i,
|
|
164
|
+
/^image[_ -]?\d*$/i,
|
|
165
|
+
/^screenshot[_ -]/i,
|
|
166
|
+
/^screen shot /i,
|
|
167
|
+
/^scan(ned)?([_ -]|$)/i,
|
|
168
|
+
/^scan \d{4}-\d{2}-\d{2} at /i,
|
|
169
|
+
/^document\s*\(\d+\)$/i,
|
|
170
|
+
/^document\d*$/i,
|
|
171
|
+
/^untitled/i,
|
|
172
|
+
/^file[_-]?\d*$/i,
|
|
173
|
+
/^download(\s*\(\d+\))?$/i,
|
|
174
|
+
/^whatsapp (image|document|video)/i,
|
|
175
|
+
/^telegram/i,
|
|
176
|
+
/^[0-9a-f]{16,}$/i,
|
|
177
|
+
/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i,
|
|
178
|
+
/^\d{8,}$/,
|
|
179
|
+
/^[\d\s._-]+$/,
|
|
180
|
+
];
|
|
181
|
+
export function nameIsUseless(originalName) {
|
|
182
|
+
const file = path.basename(originalName).trim();
|
|
183
|
+
// `path.extname('.pdf')` is '' — a leading dot means a hidden file, not an
|
|
184
|
+
// extension — so a name that is nothing but an extension has to be caught here.
|
|
185
|
+
if (/^\.[A-Za-z0-9]{1,8}$/.test(file))
|
|
186
|
+
return true;
|
|
187
|
+
const base = path.basename(file, path.extname(file)).trim();
|
|
188
|
+
if (base.length === 0)
|
|
189
|
+
return true;
|
|
190
|
+
if (USELESS_NAME.some((pattern) => pattern.test(base)))
|
|
191
|
+
return true;
|
|
192
|
+
// A name whose only content is a date says when, never what. Common for scanners.
|
|
193
|
+
const withoutDates = base.replace(/\d{4}[-_.]?\d{2}[-_.]?\d{2}|\d{2}[-_.]\d{2}[-_.]\d{4}/g, '');
|
|
194
|
+
const words = withoutDates.split(/[^A-Za-z]+/).filter((word) => word.length >= 3);
|
|
195
|
+
return words.length === 0;
|
|
196
|
+
}
|
|
197
|
+
/**
|
|
198
|
+
* The second naming guard depends on this number being meaningful, so an absent or
|
|
199
|
+
* nonsense value reads as **no** confidence rather than as full confidence. A model
|
|
200
|
+
* that forgets the field must not thereby get permission to rename.
|
|
201
|
+
*/
|
|
202
|
+
function clampConfidence(value) {
|
|
203
|
+
const parsed = typeof value === 'number' ? value : Number(value);
|
|
204
|
+
if (!Number.isFinite(parsed))
|
|
205
|
+
return 0;
|
|
206
|
+
return Math.round(Math.max(0, Math.min(1, parsed)) * 100) / 100;
|
|
207
|
+
}
|
|
208
|
+
//# sourceMappingURL=name.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"name.js","sourceRoot":"","sources":["../../src/ingest/name.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAAE,cAAc,EAAoB,MAAM,qBAAqB,CAAC;AAoCvE,MAAM,MAAM,GAAG;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kFA6BmE,CAAC;AAEnF;;wDAEwD;AACxD,MAAM,CAAC,MAAM,WAAW,GAAG,CAAC,CAAC,MAAM,CAAC;IAClC,KAAK,EAAE,CAAC,CAAC,MAAM,EAAE;IACjB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,OAAO,EAAE,CAAC,CAAC,MAAM,EAAE;IACnB,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,MAAM,EAAE,CAAC,CAAC,MAAM,EAAE,CAAC,QAAQ,EAAE;IAC7B,UAAU,EAAE,CAAC,CAAC,MAAM,EAAE;CACvB,CAAC,CAAC;AAWH,MAAM,CAAC,KAAK,UAAU,YAAY,CAAC,OAAoB;IACrD,MAAM,KAAK,GAAkB;QAC3B,KAAK,EAAE,EAAE;QACT,IAAI,EAAE,EAAE;QACR,OAAO,EAAE,EAAE;QACX,IAAI,EAAE,IAAI;QACV,eAAe,EAAE,IAAI;QACrB,UAAU,EAAE,CAAC;QACb,KAAK,EAAE,IAAI;KACZ,CAAC;IAEF,IAAI,CAAC,OAAO,CAAC,MAAM,CAAC,SAAS,EAAE,CAAC;QAC9B,OAAO,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,MAAM,CAAC,iBAAiB,IAAI,0BAA0B,EAAE,CAAC;IAC7F,CAAC;IACD,IAAI,OAAO,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,EAAE,EAAE,CAAC;QACpC,OAAO,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,2CAA2C,EAAE,CAAC;IAC1E,CAAC;IAED,iFAAiF;IACjF,2EAA2E;IAC3E,MAAM,OAAO,GAAG,OAAO,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,IAAI,CAAC,CAAC;IAC5C,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAExD,MAAM,MAAM,GAAG,MAAM,OAAO,CAAC,MAAM,CAAC,IAAI,CACtC;QACE,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,EAAE;QACnC;YACE,IAAI,EAAE,MAAM;YACZ,OAAO,EAAE,qBAAqB,OAAO,IAAI,YAAY,0BAA0B,OAAO,EAAE;SACzF;KACF,EACD,EAAE,MAAM,EAAE,WAAW,EAAE,SAAS,EAAE,GAAG,EAAE,CACxC,CAAC;IACF,IAAI,CAAC,MAAM,CAAC,EAAE,IAAI,CAAC,MAAM,CAAC,KAAK;QAAE,OAAO,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,MAAM,CAAC,KAAK,IAAI,eAAe,EAAE,CAAC;IAE7F,MAAM,MAAM,GAAG,cAAc,CAA0B,MAAM,CAAC,KAAK,CAAC,CAAC;IACrE,IAAI,CAAC,MAAM;QAAE,OAAO,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,kCAAkC,EAAE,CAAC;IAE5E,MAAM,IAAI,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IACpC,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC,CAAC;IACnD,IAAI,CAAC,IAAI,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,GAAG,KAAK,EAAE,KAAK,EAAE,yCAAyC,EAAE,CAAC;IAE3F,MAAM,SAAS,GAAG,SAAS,CAAC,MAAM,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAChD,OAAO;QACL,KAAK;QACL,IAAI;QACJ,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,GAAG,CAAC,IAAI,EAAE;QAC7C,IAAI,EAAE,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC;QAC5B,6EAA6E;QAC7E,gEAAgE;QAChE,eAAe,EAAE,SAAS,IAAI,OAAO,CAAC,OAAO,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI;QACpF,UAAU,EAAE,eAAe,CAAC,MAAM,CAAC,UAAU,CAAC;QAC9C,KAAK,EAAE,IAAI;KACZ,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,MAAM,CAAC,KAAoB;IACzC,IAAI,KAAK,KAAK,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACrD,IAAI,CAAC,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtC,OAAO,KAAK,CAAC,OAAO,CAAC,QAAQ,EAAE,GAAG,CAAC,CAAC;AACtC,CAAC;AAED,SAAS,SAAS,CAAC,KAAc,EAAE,GAAW;IAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3C,MAAM,OAAO,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,OAAO,CAAC,MAAM,EAAE,GAAG,CAAC,CAAC;IAClD,OAAO,OAAO,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC3D,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3C,MAAM,IAAI,GAAG,KAAK,CAAC,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC9D,OAAO,qBAAqB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AACxD,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,SAAS,CAAC,KAAc;IACtC,IAAI,OAAO,KAAK,KAAK,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3C,MAAM,IAAI,GAAG,KAAK;SACf,IAAI,EAAE;SACN,WAAW,EAAE;SACb,KAAK,CAAC,GAAG,CAAC;SACV,GAAG,EAAG;SACN,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC/B,SAAS,CAAC,MAAM,CAAC;SACjB,OAAO,CAAC,kBAAkB,EAAE,EAAE,CAAC;SAC/B,OAAO,CAAC,aAAa,EAAE,GAAG,CAAC;SAC3B,OAAO,CAAC,UAAU,EAAE,EAAE,CAAC;SACvB,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC;SACZ,OAAO,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;IACtB,OAAO,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC;AACxC,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,YAAY,GAAG;IACnB,gBAAgB;IAChB,gBAAgB;IAChB,YAAY;IACZ,YAAY;IACZ,6EAA6E;IAC7E,sCAAsC;IACtC,qCAAqC;IACrC,8BAA8B;IAC9B,wBAAwB;IACxB,mBAAmB;IACnB,mBAAmB;IACnB,mBAAmB;IACnB,gBAAgB;IAChB,uBAAuB;IACvB,8BAA8B;IAC9B,uBAAuB;IACvB,gBAAgB;IAChB,YAAY;IACZ,iBAAiB;IACjB,0BAA0B;IAC1B,mCAAmC;IACnC,YAAY;IACZ,kBAAkB;IAClB,iEAAiE;IACjE,UAAU;IACV,cAAc;CACf,CAAC;AAEF,MAAM,UAAU,aAAa,CAAC,YAAoB;IAChD,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,YAAY,CAAC,CAAC,IAAI,EAAE,CAAC;IAChD,2EAA2E;IAC3E,gFAAgF;IAChF,IAAI,sBAAsB,CAAC,IAAI,CAAC,IAAI,CAAC;QAAE,OAAO,IAAI,CAAC;IAEnD,MAAM,IAAI,GAAG,IAAI,CAAC,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5D,IAAI,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,IAAI,CAAC;IACnC,IAAI,YAAY,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAAE,OAAO,IAAI,CAAC;IAEpE,kFAAkF;IAClF,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,wDAAwD,EAAE,EAAE,CAAC,CAAC;IAChG,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,CAAC,MAAM,IAAI,CAAC,CAAC,CAAC;IAClF,OAAO,KAAK,CAAC,MAAM,KAAK,CAAC,CAAC;AAC5B,CAAC;AAED;;;;GAIG;AACH,SAAS,eAAe,CAAC,KAAc;IACrC,MAAM,MAAM,GAAG,OAAO,KAAK,KAAK,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;IACjE,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,MAAM,CAAC;QAAE,OAAO,CAAC,CAAC;IACvC,OAAO,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC,GAAG,GAAG,CAAC,GAAG,GAAG,CAAC;AAClE,CAAC"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Scanning a passport gives you `passport.pdf` and `passport-2.pdf`. They are not two
|
|
3
|
+
* documents — they are one document someone's scanner cut in half.
|
|
4
|
+
*
|
|
5
|
+
* Treating them separately produces two pages, two summaries and two half-answers, and
|
|
6
|
+
* "page 2" of the second file is really page 5 of the passport. So files that differ only
|
|
7
|
+
* by a trailing `-<n>` are read as **parts of one document**: one owning page, one summary,
|
|
8
|
+
* and page numbers that run through the whole thing.
|
|
9
|
+
*
|
|
10
|
+
* The rule is deliberately narrow, because the cost of a wrong guess is two unrelated
|
|
11
|
+
* documents welded together:
|
|
12
|
+
*
|
|
13
|
+
* - The extension must match. `passport.jpg` beside `passport.pdf` is a second rendition of
|
|
14
|
+
* the same thing, not its second half, and merging their text would interleave two
|
|
15
|
+
* readings of one page.
|
|
16
|
+
* - The suffix must be one or two digits **not preceded by a digit**. `bill-2026.pdf` is a
|
|
17
|
+
* year and `invoice-100.pdf` is an invoice number — but the case that actually bit was
|
|
18
|
+
* `waternet-annual-bill-2026-07-28.pdf`, read as part 28 of a bill that does not exist.
|
|
19
|
+
* Two bills dated `-27` and `-28` would have been welded into one document.
|
|
20
|
+
* - Part one must exist on disk. A lone `-<n>` file is its own document, whatever its name
|
|
21
|
+
* suggests, because the alternative is inventing a document out of a filename.
|
|
22
|
+
* - A content-addressed attachment (`<page>-<8 hex>.<ext>`) is never a part. Its suffix is
|
|
23
|
+
* a hash, and Akno wrote it.
|
|
24
|
+
*/
|
|
25
|
+
/**
|
|
26
|
+
* The other thing a second file beside a document can be.
|
|
27
|
+
*
|
|
28
|
+
* The part rule above excludes a differing extension because `passport.jpg` beside
|
|
29
|
+
* `passport.pdf` is "a second rendition of the same thing, not its second half". That
|
|
30
|
+
* category was named and then had nowhere to go. This is where it goes: `contract.pdf.txt`
|
|
31
|
+
* holds the *same* text as `contract.pdf`, in a format anything can open.
|
|
32
|
+
*
|
|
33
|
+
* The distinction is not cosmetic. Parts concatenate — `read` joins their text and page
|
|
34
|
+
* numbers run through the whole group — while a rendition must never be joined to what it
|
|
35
|
+
* renders, or a contract's every phrase comes back twice.
|
|
36
|
+
*
|
|
37
|
+
* Two spellings, because both occur. Akno writes `contract.pdf.txt`; a person who ran
|
|
38
|
+
* `pdftotext` before Akno existed wrote `contract.txt` beside `contract.pdf`, and that
|
|
39
|
+
* second one is the case that was quietly costing double hits on every phrase in the file.
|
|
40
|
+
*
|
|
41
|
+
* The rule stays as narrow as the part rule, and for the same reasons:
|
|
42
|
+
*
|
|
43
|
+
* - **Only a `.txt` is ever a rendition.** `passport.jpg` beside `passport.pdf` is two
|
|
44
|
+
* renderings with no way to say which is the document, so neither is folded into the other.
|
|
45
|
+
* - **The source must exist, and be a format text is extracted *from*.** A `.txt` beside a
|
|
46
|
+
* `.csv` is two text files, not a rendering of one.
|
|
47
|
+
* - **An ambiguous stem is refused.** `contract.txt` beside both `contract.pdf` and
|
|
48
|
+
* `contract.docx` names no one document, and picking either would attach the text of one
|
|
49
|
+
* file to a different file's name.
|
|
50
|
+
* - `.txt.txt` is refused. Whatever it is, it is not a rendering of a text file.
|
|
51
|
+
*/
|
|
52
|
+
export interface DocumentRendition {
|
|
53
|
+
/** The rel_path of the file this one renders. */
|
|
54
|
+
source: string;
|
|
55
|
+
}
|
|
56
|
+
export interface RenditionOptions {
|
|
57
|
+
/** Names of the files in a directory, for finding a same-stem sibling. */
|
|
58
|
+
entries: (directory: string) => string[];
|
|
59
|
+
}
|
|
60
|
+
export declare function documentRendition(relPath: string, options: RenditionOptions): DocumentRendition | null;
|
|
61
|
+
export interface DocumentPart {
|
|
62
|
+
/**
|
|
63
|
+
* The rel_path of part one — the group's identity, whether or not that file exists. Files
|
|
64
|
+
* sharing one `groupKey` are one document.
|
|
65
|
+
*/
|
|
66
|
+
groupKey: string;
|
|
67
|
+
/** 1 for the bare file, `n` for `-<n>`. Parts are ordered by this. */
|
|
68
|
+
part: number;
|
|
69
|
+
}
|
|
70
|
+
export interface PartOptions {
|
|
71
|
+
/**
|
|
72
|
+
* Whether the group's part one exists. Without this check a date-suffixed name invents a
|
|
73
|
+
* document: the guard is that a part implies something to be a part *of*.
|
|
74
|
+
*/
|
|
75
|
+
hasPartOne?: (groupKey: string) => boolean;
|
|
76
|
+
}
|
|
77
|
+
export declare function documentPart(relPath: string, options?: PartOptions): DocumentPart;
|
|
78
|
+
//# sourceMappingURL=parts.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parts.d.ts","sourceRoot":"","sources":["../../src/ingest/parts.ts"],"names":[],"mappings":"AAEA;;;;;;;;;;;;;;;;;;;;;;;GAuBG;AAEH;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AACH,MAAM,WAAW,iBAAiB;IAChC,iDAAiD;IACjD,MAAM,EAAE,MAAM,CAAC;CAChB;AAED,MAAM,WAAW,gBAAgB;IAC/B,0EAA0E;IAC1E,OAAO,EAAE,CAAC,SAAS,EAAE,MAAM,KAAK,MAAM,EAAE,CAAC;CAC1C;AAKD,wBAAgB,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,gBAAgB,GAAG,iBAAiB,GAAG,IAAI,CAmBtG;AAaD,MAAM,WAAW,YAAY;IAC3B;;;OAGG;IACH,QAAQ,EAAE,MAAM,CAAC;IACjB,sEAAsE;IACtE,IAAI,EAAE,MAAM,CAAC;CACd;AAID,MAAM,WAAW,WAAW;IAC1B;;;OAGG;IACH,UAAU,CAAC,EAAE,CAAC,QAAQ,EAAE,MAAM,KAAK,OAAO,CAAC;CAC5C;AAED,wBAAgB,YAAY,CAAC,OAAO,EAAE,MAAM,EAAE,OAAO,GAAE,WAAgB,GAAG,YAAY,CAoBrF"}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
/** Formats whose text is a copy of somebody else's, so a `.txt` of one adds nothing. */
|
|
3
|
+
const PLAIN = new Set(['.txt', '.md', '.markdown', '.csv', '.tsv', '.json', '.yaml', '.yml', '.log']);
|
|
4
|
+
export function documentRendition(relPath, options) {
|
|
5
|
+
const normalized = relPath.replaceAll('\\', '/');
|
|
6
|
+
if (path.posix.extname(normalized).toLowerCase() !== '.txt')
|
|
7
|
+
return null;
|
|
8
|
+
const directory = path.posix.dirname(normalized);
|
|
9
|
+
const stem = path.posix.basename(normalized.slice(0, -'.txt'.length));
|
|
10
|
+
const siblings = options.entries(directory);
|
|
11
|
+
const inDirectory = (name) => (directory === '.' ? name : `${directory}/${name}`);
|
|
12
|
+
// Asked of the folder rather than of the name. Whether a stem "has an extension" cannot be
|
|
13
|
+
// decided by looking for a dot: `Rental Agreement … A. N. Marlow … Aug 5 2031` has four, and
|
|
14
|
+
// none of them starts an extension. Whether a file of that exact name is *there* is a fact.
|
|
15
|
+
if (siblings.includes(stem)) {
|
|
16
|
+
return isExtractable(stem) ? { source: inDirectory(stem) } : null;
|
|
17
|
+
}
|
|
18
|
+
// Otherwise the folder has to hold exactly one document this could be the text of.
|
|
19
|
+
const candidates = siblings.filter((name) => isExtractable(name) && withoutExtension(name) === stem).sort();
|
|
20
|
+
return candidates.length === 1 ? { source: inDirectory(candidates[0]) } : null;
|
|
21
|
+
}
|
|
22
|
+
/** A format text is extracted *from*, rather than one that is already text. */
|
|
23
|
+
function isExtractable(name) {
|
|
24
|
+
const extension = path.posix.extname(name).toLowerCase();
|
|
25
|
+
return extension.length > 0 && !PLAIN.has(extension);
|
|
26
|
+
}
|
|
27
|
+
function withoutExtension(name) {
|
|
28
|
+
const extension = path.posix.extname(name);
|
|
29
|
+
return extension ? name.slice(0, -extension.length) : name;
|
|
30
|
+
}
|
|
31
|
+
const PART_SUFFIX = /^(.*[^\d-])-(\d{1,2})$/;
|
|
32
|
+
export function documentPart(relPath, options = {}) {
|
|
33
|
+
const normalized = relPath.replaceAll('\\', '/');
|
|
34
|
+
const extension = path.posix.extname(normalized);
|
|
35
|
+
const base = path.posix.basename(normalized, extension);
|
|
36
|
+
const directory = path.posix.dirname(normalized);
|
|
37
|
+
const match = PART_SUFFIX.exec(base);
|
|
38
|
+
if (!match)
|
|
39
|
+
return { groupKey: normalized, part: 1 };
|
|
40
|
+
const part = Number(match[2]);
|
|
41
|
+
// 0 is not a part number anyone writes, and a leading-zero form (`-02`) is a page label
|
|
42
|
+
// from a scanner rather than a sequence Akno should reorder.
|
|
43
|
+
if (part < 1 || match[2].startsWith('0'))
|
|
44
|
+
return { groupKey: normalized, part: 1 };
|
|
45
|
+
const stem = match[1];
|
|
46
|
+
const groupBase = `${stem}${extension}`;
|
|
47
|
+
const groupKey = directory === '.' ? groupBase : `${directory}/${groupBase}`;
|
|
48
|
+
if (options.hasPartOne && !options.hasPartOne(groupKey))
|
|
49
|
+
return { groupKey: normalized, part: 1 };
|
|
50
|
+
return { groupKey, part };
|
|
51
|
+
}
|
|
52
|
+
//# sourceMappingURL=parts.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parts.js","sourceRoot":"","sources":["../../src/ingest/parts.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAgE7B,wFAAwF;AACxF,MAAM,KAAK,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,KAAK,EAAE,WAAW,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,OAAO,EAAE,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAEtG,MAAM,UAAU,iBAAiB,CAAC,OAAe,EAAE,OAAyB;IAC1E,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACjD,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC,WAAW,EAAE,KAAK,MAAM;QAAE,OAAO,IAAI,CAAC;IAEzE,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC;IACtE,MAAM,QAAQ,GAAG,OAAO,CAAC,OAAO,CAAC,SAAS,CAAC,CAAC;IAC5C,MAAM,WAAW,GAAG,CAAC,IAAY,EAAU,EAAE,CAAC,CAAC,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,GAAG,SAAS,IAAI,IAAI,EAAE,CAAC,CAAC;IAElG,2FAA2F;IAC3F,6FAA6F;IAC7F,4FAA4F;IAC5F,IAAI,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,CAAC;QAC5B,OAAO,aAAa,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;IACpE,CAAC;IAED,mFAAmF;IACnF,MAAM,UAAU,GAAG,QAAQ,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,aAAa,CAAC,IAAI,CAAC,IAAI,gBAAgB,CAAC,IAAI,CAAC,KAAK,IAAI,CAAC,CAAC,IAAI,EAAE,CAAC;IAC5G,OAAO,UAAU,CAAC,MAAM,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,WAAW,CAAC,UAAU,CAAC,CAAC,CAAE,CAAC,EAAE,CAAC,CAAC,CAAC,IAAI,CAAC;AAClF,CAAC;AAED,+EAA+E;AAC/E,SAAS,aAAa,CAAC,IAAY;IACjC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,WAAW,EAAE,CAAC;IACzD,OAAO,SAAS,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AACvD,CAAC;AAED,SAAS,gBAAgB,CAAC,IAAY;IACpC,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC3C,OAAO,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AAC7D,CAAC;AAYD,MAAM,WAAW,GAAG,wBAAwB,CAAC;AAU7C,MAAM,UAAU,YAAY,CAAC,OAAe,EAAE,UAAuB,EAAE;IACrE,MAAM,UAAU,GAAG,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;IACjD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IACjD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;IACxD,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,UAAU,CAAC,CAAC;IAEjD,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrC,IAAI,CAAC,KAAK;QAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAErD,MAAM,IAAI,GAAG,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;IAC9B,wFAAwF;IACxF,6DAA6D;IAC7D,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,CAAC,CAAC,CAAE,CAAC,UAAU,CAAC,GAAG,CAAC;QAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAEpF,MAAM,IAAI,GAAG,KAAK,CAAC,CAAC,CAAE,CAAC;IACvB,MAAM,SAAS,GAAG,GAAG,IAAI,GAAG,SAAS,EAAE,CAAC;IACxC,MAAM,QAAQ,GAAG,SAAS,KAAK,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,GAAG,SAAS,IAAI,SAAS,EAAE,CAAC;IAE7E,IAAI,OAAO,CAAC,UAAU,IAAI,CAAC,OAAO,CAAC,UAAU,CAAC,QAAQ,CAAC;QAAE,OAAO,EAAE,QAAQ,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC,EAAE,CAAC;IAClG,OAAO,EAAE,QAAQ,EAAE,IAAI,EAAE,CAAC;AAC5B,CAAC"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import type { Extraction } from './extract.ts';
|
|
2
|
+
/**
|
|
3
|
+
* The extracted text, written beside the file it came from.
|
|
4
|
+
*
|
|
5
|
+
* The text of a stored document already exists — `documents.text`, read once on arrival —
|
|
6
|
+
* and `read({document})` hands back every character of it. What did not exist is a way for
|
|
7
|
+
* anything that is *not* Akno to see it: a `grep`, an editor, a git diff, an agent that
|
|
8
|
+
* has a folder and no socket. The page beside a stored PDF is two sentences on purpose, and
|
|
9
|
+
* two sentences cannot hold a nine-page lease, so the text had nowhere on disk to live and
|
|
10
|
+
* the only way back to it was to OCR the PDF again.
|
|
11
|
+
*
|
|
12
|
+
* A rendition is that file. It is derived, regenerable and never authoritative: the source
|
|
13
|
+
* bytes are the document, `documents.text` is what search reads, and this is a convenience
|
|
14
|
+
* copy that says so in its own first three lines.
|
|
15
|
+
*/
|
|
16
|
+
export interface RenditionSource {
|
|
17
|
+
relPath: string;
|
|
18
|
+
text: string;
|
|
19
|
+
pageCount: number | null;
|
|
20
|
+
ocr: boolean;
|
|
21
|
+
via: Extraction['via'];
|
|
22
|
+
confidence: number | null;
|
|
23
|
+
}
|
|
24
|
+
export interface RenditionGate {
|
|
25
|
+
minChars: number;
|
|
26
|
+
/** The `ingest` verb of the folder rule in force for the owning page. */
|
|
27
|
+
ingestRule: 'page' | 'document' | 'file' | 'auto' | 'ignore' | undefined;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Whether a document earns a file beside it — and, when it does not, why not.
|
|
31
|
+
*
|
|
32
|
+
* An allowlist on `via` rather than a denylist, so a format added later has to say out loud
|
|
33
|
+
* that its text is the document's own words before those words are written to disk under
|
|
34
|
+
* the document's name.
|
|
35
|
+
*/
|
|
36
|
+
export declare function renditionWanted(source: RenditionSource, gate: RenditionGate): {
|
|
37
|
+
write: boolean;
|
|
38
|
+
reason: string;
|
|
39
|
+
};
|
|
40
|
+
/**
|
|
41
|
+
* Whether Akno wrote this, judged from the file itself.
|
|
42
|
+
*
|
|
43
|
+
* The same question `looksLikeLedger` asks of a `timeline.md`, and answered the same way:
|
|
44
|
+
* a reserved path that exists and is not what Akno expects is left completely alone.
|
|
45
|
+
* Somebody who opened `contract.pdf.txt` and fixed three OCR mistakes by hand did something
|
|
46
|
+
* useful, and reverting it on the next index pass is not a recoverable mistake.
|
|
47
|
+
*/
|
|
48
|
+
export declare function looksLikeRendition(head: string): boolean;
|
|
49
|
+
/**
|
|
50
|
+
* A provenance header, a blank line, then the text exactly as it is stored.
|
|
51
|
+
*
|
|
52
|
+
* The header is three lines against tens of thousands of characters, and it buys the thing
|
|
53
|
+
* `provenanceLines` exists for: text recognised from a scan at 0.87 confidence is a good
|
|
54
|
+
* guess, not a transcript, and a file that does not say so invites being quoted as one.
|
|
55
|
+
*
|
|
56
|
+
* **No page markers.** The temptation is real — a citation wants "page 9" — but the per-page
|
|
57
|
+
* sections and the joined text are built separately during extraction and are not the same
|
|
58
|
+
* string for a PDF with a text layer. A rendition assembled from sections would therefore
|
|
59
|
+
* hold different bytes from the text `read` returns and the summary was written from: one
|
|
60
|
+
* document with two texts that can quietly disagree. Page numbers already live where they
|
|
61
|
+
* survive being checked, on the chunks recall cites.
|
|
62
|
+
*/
|
|
63
|
+
export declare function renditionBody(source: RenditionSource): string;
|
|
64
|
+
/**
|
|
65
|
+
* Where the text of a document goes: the document's own name, with `.txt` for an extension.
|
|
66
|
+
*
|
|
67
|
+
* `contract.txt` beside `contract.pdf` rather than `contract.pdf.txt`, because a document is
|
|
68
|
+
* one thing and its stem is its name — the same reason `passport.pdf` and `passport-2.pdf`
|
|
69
|
+
* are one document. Appending instead of replacing produces a second file beside anyone who
|
|
70
|
+
* already ran `pdftotext`, which is two copies of one text under two names.
|
|
71
|
+
*
|
|
72
|
+
* The reader accepts both spellings; only this one is written.
|
|
73
|
+
*/
|
|
74
|
+
export declare function renditionPathFor(relPath: string): string;
|
|
75
|
+
//# sourceMappingURL=rendition.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rendition.d.ts","sourceRoot":"","sources":["../../src/ingest/rendition.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,cAAc,CAAC;AAE/C;;;;;;;;;;;;;GAaG;AAEH,MAAM,WAAW,eAAe;IAC9B,OAAO,EAAE,MAAM,CAAC;IAChB,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,GAAG,EAAE,OAAO,CAAC;IACb,GAAG,EAAE,UAAU,CAAC,KAAK,CAAC,CAAC;IACvB,UAAU,EAAE,MAAM,GAAG,IAAI,CAAC;CAC3B;AAED,MAAM,WAAW,aAAa;IAC5B,QAAQ,EAAE,MAAM,CAAC;IACjB,yEAAyE;IACzE,UAAU,EAAE,MAAM,GAAG,UAAU,GAAG,MAAM,GAAG,MAAM,GAAG,QAAQ,GAAG,SAAS,CAAC;CAC1E;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe,CAC7B,MAAM,EAAE,eAAe,EACvB,IAAI,EAAE,aAAa,GAClB;IAAE,KAAK,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,MAAM,CAAA;CAAE,CA2BpC;AAKD;;;;;;;GAOG;AACH,wBAAgB,kBAAkB,CAAC,IAAI,EAAE,MAAM,GAAG,OAAO,CAExD;AAED;;;;;;;;;;;;;GAaG;AACH,wBAAgB,aAAa,CAAC,MAAM,EAAE,eAAe,GAAG,MAAM,CAmB7D;AAED;;;;;;;;;GASG;AACH,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,MAAM,GAAG,MAAM,CAGxD"}
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import path from 'node:path';
|
|
2
|
+
/**
|
|
3
|
+
* Whether a document earns a file beside it — and, when it does not, why not.
|
|
4
|
+
*
|
|
5
|
+
* An allowlist on `via` rather than a denylist, so a format added later has to say out loud
|
|
6
|
+
* that its text is the document's own words before those words are written to disk under
|
|
7
|
+
* the document's name.
|
|
8
|
+
*/
|
|
9
|
+
export function renditionWanted(source, gate) {
|
|
10
|
+
if (gate.ingestRule === 'ignore' || gate.ingestRule === 'file') {
|
|
11
|
+
// `ingest: "file"` means "index the bytes, write me no pages here". A rendition is a
|
|
12
|
+
// file written into that folder, so the rule covers it for the same reason.
|
|
13
|
+
return { write: false, reason: `the folder rule says ingest: "${gate.ingestRule}"` };
|
|
14
|
+
}
|
|
15
|
+
if (source.text.length === 0)
|
|
16
|
+
return { write: false, reason: 'nothing was extracted' };
|
|
17
|
+
// The file already *is* its text. A byte-for-byte copy under a longer name is not a
|
|
18
|
+
// convenience, it is a second thing to keep in step.
|
|
19
|
+
if (source.via === 'plain')
|
|
20
|
+
return { write: false, reason: 'the file is already text' };
|
|
21
|
+
// A model looking at a photograph and saying what it sees is not the file's words. Writing
|
|
22
|
+
// that into `garden.jpg.txt` would claim it is, and a reader deciding whether to trust a
|
|
23
|
+
// number has no way left to tell.
|
|
24
|
+
if (source.via === 'vision') {
|
|
25
|
+
return { write: false, reason: "the text is a model's description of an image, not the file's words" };
|
|
26
|
+
}
|
|
27
|
+
if (source.via === 'none')
|
|
28
|
+
return { write: false, reason: 'the text could not be read' };
|
|
29
|
+
if (source.text.length < gate.minChars) {
|
|
30
|
+
return {
|
|
31
|
+
write: false,
|
|
32
|
+
reason: `${source.text.length} characters is under ingest.text_rendition_min_chars (${gate.minChars})`,
|
|
33
|
+
};
|
|
34
|
+
}
|
|
35
|
+
return { write: true, reason: 'the text is the document’s own and does not fit on its page' };
|
|
36
|
+
}
|
|
37
|
+
/** The first thing in every rendition, and the only claim of authorship there is. */
|
|
38
|
+
const HEADER = '# Extracted text of ';
|
|
39
|
+
/**
|
|
40
|
+
* Whether Akno wrote this, judged from the file itself.
|
|
41
|
+
*
|
|
42
|
+
* The same question `looksLikeLedger` asks of a `timeline.md`, and answered the same way:
|
|
43
|
+
* a reserved path that exists and is not what Akno expects is left completely alone.
|
|
44
|
+
* Somebody who opened `contract.pdf.txt` and fixed three OCR mistakes by hand did something
|
|
45
|
+
* useful, and reverting it on the next index pass is not a recoverable mistake.
|
|
46
|
+
*/
|
|
47
|
+
export function looksLikeRendition(head) {
|
|
48
|
+
return head.startsWith(HEADER);
|
|
49
|
+
}
|
|
50
|
+
/**
|
|
51
|
+
* A provenance header, a blank line, then the text exactly as it is stored.
|
|
52
|
+
*
|
|
53
|
+
* The header is three lines against tens of thousands of characters, and it buys the thing
|
|
54
|
+
* `provenanceLines` exists for: text recognised from a scan at 0.87 confidence is a good
|
|
55
|
+
* guess, not a transcript, and a file that does not say so invites being quoted as one.
|
|
56
|
+
*
|
|
57
|
+
* **No page markers.** The temptation is real — a citation wants "page 9" — but the per-page
|
|
58
|
+
* sections and the joined text are built separately during extraction and are not the same
|
|
59
|
+
* string for a PDF with a text layer. A rendition assembled from sections would therefore
|
|
60
|
+
* hold different bytes from the text `read` returns and the summary was written from: one
|
|
61
|
+
* document with two texts that can quietly disagree. Page numbers already live where they
|
|
62
|
+
* survive being checked, on the chunks recall cites.
|
|
63
|
+
*/
|
|
64
|
+
export function renditionBody(source) {
|
|
65
|
+
const name = path.posix.basename(source.relPath.replaceAll('\\', '/'));
|
|
66
|
+
const pages = source.pageCount !== null ? `${source.pageCount} page${source.pageCount === 1 ? '' : 's'}, ` : '';
|
|
67
|
+
const confidence = source.confidence !== null ? ` at ${source.confidence.toFixed(2)} mean confidence` : '';
|
|
68
|
+
const how = source.via === 'ocr'
|
|
69
|
+
? `recognised by OCR${confidence}, not typed`
|
|
70
|
+
: source.via === 'textutil'
|
|
71
|
+
? 'converted from the original format'
|
|
72
|
+
: "read from the file's own text layer";
|
|
73
|
+
return (`${HEADER}${name}\n` +
|
|
74
|
+
`# ${pages}${how}.\n` +
|
|
75
|
+
`# Written by Akno from the file beside it. Edits here are overwritten; edit the source.\n` +
|
|
76
|
+
`\n${source.text}\n`);
|
|
77
|
+
}
|
|
78
|
+
/**
|
|
79
|
+
* Where the text of a document goes: the document's own name, with `.txt` for an extension.
|
|
80
|
+
*
|
|
81
|
+
* `contract.txt` beside `contract.pdf` rather than `contract.pdf.txt`, because a document is
|
|
82
|
+
* one thing and its stem is its name — the same reason `passport.pdf` and `passport-2.pdf`
|
|
83
|
+
* are one document. Appending instead of replacing produces a second file beside anyone who
|
|
84
|
+
* already ran `pdftotext`, which is two copies of one text under two names.
|
|
85
|
+
*
|
|
86
|
+
* The reader accepts both spellings; only this one is written.
|
|
87
|
+
*/
|
|
88
|
+
export function renditionPathFor(relPath) {
|
|
89
|
+
const extension = path.posix.extname(relPath.replaceAll('\\', '/'));
|
|
90
|
+
return `${extension ? relPath.slice(0, -extension.length) : relPath}.txt`;
|
|
91
|
+
}
|
|
92
|
+
//# sourceMappingURL=rendition.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"rendition.js","sourceRoot":"","sources":["../../src/ingest/rendition.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAC;AAiC7B;;;;;;GAMG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAuB,EACvB,IAAmB;IAEnB,IAAI,IAAI,CAAC,UAAU,KAAK,QAAQ,IAAI,IAAI,CAAC,UAAU,KAAK,MAAM,EAAE,CAAC;QAC/D,qFAAqF;QACrF,4EAA4E;QAC5E,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,iCAAiC,IAAI,CAAC,UAAU,GAAG,EAAE,CAAC;IACvF,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC;QAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,uBAAuB,EAAE,CAAC;IAEvF,oFAAoF;IACpF,qDAAqD;IACrD,IAAI,MAAM,CAAC,GAAG,KAAK,OAAO;QAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,0BAA0B,EAAE,CAAC;IAExF,2FAA2F;IAC3F,yFAAyF;IACzF,kCAAkC;IAClC,IAAI,MAAM,CAAC,GAAG,KAAK,QAAQ,EAAE,CAAC;QAC5B,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,qEAAqE,EAAE,CAAC;IACzG,CAAC;IACD,IAAI,MAAM,CAAC,GAAG,KAAK,MAAM;QAAE,OAAO,EAAE,KAAK,EAAE,KAAK,EAAE,MAAM,EAAE,4BAA4B,EAAE,CAAC;IAEzF,IAAI,MAAM,CAAC,IAAI,CAAC,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;QACvC,OAAO;YACL,KAAK,EAAE,KAAK;YACZ,MAAM,EAAE,GAAG,MAAM,CAAC,IAAI,CAAC,MAAM,yDAAyD,IAAI,CAAC,QAAQ,GAAG;SACvG,CAAC;IACJ,CAAC;IACD,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAE,6DAA6D,EAAE,CAAC;AAChG,CAAC;AAED,qFAAqF;AACrF,MAAM,MAAM,GAAG,sBAAsB,CAAC;AAEtC;;;;;;;GAOG;AACH,MAAM,UAAU,kBAAkB,CAAC,IAAY;IAC7C,OAAO,IAAI,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;AACjC,CAAC;AAED;;;;;;;;;;;;;GAaG;AACH,MAAM,UAAU,aAAa,CAAC,MAAuB;IACnD,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;IACvE,MAAM,KAAK,GACT,MAAM,CAAC,SAAS,KAAK,IAAI,CAAC,CAAC,CAAC,GAAG,MAAM,CAAC,SAAS,QAAQ,MAAM,CAAC,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,GAAG,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC;IACpG,MAAM,UAAU,GAAG,MAAM,CAAC,UAAU,KAAK,IAAI,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC,CAAC,kBAAkB,CAAC,CAAC,CAAC,EAAE,CAAC;IAE3G,MAAM,GAAG,GACP,MAAM,CAAC,GAAG,KAAK,KAAK;QAClB,CAAC,CAAC,oBAAoB,UAAU,aAAa;QAC7C,CAAC,CAAC,MAAM,CAAC,GAAG,KAAK,UAAU;YACzB,CAAC,CAAC,oCAAoC;YACtC,CAAC,CAAC,qCAAqC,CAAC;IAE9C,OAAO,CACL,GAAG,MAAM,GAAG,IAAI,IAAI;QACpB,KAAK,KAAK,GAAG,GAAG,KAAK;QACrB,2FAA2F;QAC3F,KAAK,MAAM,CAAC,IAAI,IAAI,CACrB,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,gBAAgB,CAAC,OAAe;IAC9C,MAAM,SAAS,GAAG,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,UAAU,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC,CAAC;IACpE,OAAO,GAAG,SAAS,CAAC,CAAC,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,OAAO,MAAM,CAAC;AAC5E,CAAC"}
|