@tenphi/akno-core 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +83 -0
- package/README.md +26 -0
- package/config/default.jsonc +548 -0
- package/dist/bench/answer-corpus.d.ts +48 -0
- package/dist/bench/answer-corpus.d.ts.map +1 -0
- package/dist/bench/answer-corpus.js +491 -0
- package/dist/bench/answer-corpus.js.map +1 -0
- package/dist/bench/answer.d.ts +156 -0
- package/dist/bench/answer.d.ts.map +1 -0
- package/dist/bench/answer.js +608 -0
- package/dist/bench/answer.js.map +1 -0
- package/dist/bench/auto-recall-answer-corpus.d.ts +21 -0
- package/dist/bench/auto-recall-answer-corpus.d.ts.map +1 -0
- package/dist/bench/auto-recall-answer-corpus.js +245 -0
- package/dist/bench/auto-recall-answer-corpus.js.map +1 -0
- package/dist/bench/auto-recall-answer.d.ts +198 -0
- package/dist/bench/auto-recall-answer.d.ts.map +1 -0
- package/dist/bench/auto-recall-answer.js +720 -0
- package/dist/bench/auto-recall-answer.js.map +1 -0
- package/dist/bench/auto-recall-corpus.d.ts +45 -0
- package/dist/bench/auto-recall-corpus.d.ts.map +1 -0
- package/dist/bench/auto-recall-corpus.js +221 -0
- package/dist/bench/auto-recall-corpus.js.map +1 -0
- package/dist/bench/auto-recall.d.ts +171 -0
- package/dist/bench/auto-recall.d.ts.map +1 -0
- package/dist/bench/auto-recall.js +651 -0
- package/dist/bench/auto-recall.js.map +1 -0
- package/dist/bench/entity-resolution.d.ts +37 -0
- package/dist/bench/entity-resolution.d.ts.map +1 -0
- package/dist/bench/entity-resolution.js +121 -0
- package/dist/bench/entity-resolution.js.map +1 -0
- package/dist/bench/graph.d.ts +57 -0
- package/dist/bench/graph.d.ts.map +1 -0
- package/dist/bench/graph.js +573 -0
- package/dist/bench/graph.js.map +1 -0
- package/dist/bench/llm-ranking-probe.d.ts +27 -0
- package/dist/bench/llm-ranking-probe.d.ts.map +1 -0
- package/dist/bench/llm-ranking-probe.js +105 -0
- package/dist/bench/llm-ranking-probe.js.map +1 -0
- package/dist/bench/merge-discovery-corpus.d.ts +26 -0
- package/dist/bench/merge-discovery-corpus.d.ts.map +1 -0
- package/dist/bench/merge-discovery-corpus.js +114 -0
- package/dist/bench/merge-discovery-corpus.js.map +1 -0
- package/dist/bench/merge-discovery-review.d.ts +81 -0
- package/dist/bench/merge-discovery-review.d.ts.map +1 -0
- package/dist/bench/merge-discovery-review.js +149 -0
- package/dist/bench/merge-discovery-review.js.map +1 -0
- package/dist/bench/merge-discovery.d.ts +122 -0
- package/dist/bench/merge-discovery.d.ts.map +1 -0
- package/dist/bench/merge-discovery.js +390 -0
- package/dist/bench/merge-discovery.js.map +1 -0
- package/dist/bench/mixed-retrieval.d.ts +35 -0
- package/dist/bench/mixed-retrieval.d.ts.map +1 -0
- package/dist/bench/mixed-retrieval.js +355 -0
- package/dist/bench/mixed-retrieval.js.map +1 -0
- package/dist/bench/ranking-corpus.d.ts +29 -0
- package/dist/bench/ranking-corpus.d.ts.map +1 -0
- package/dist/bench/ranking-corpus.js +514 -0
- package/dist/bench/ranking-corpus.js.map +1 -0
- package/dist/bench/ranking-end-to-end.d.ts +112 -0
- package/dist/bench/ranking-end-to-end.d.ts.map +1 -0
- package/dist/bench/ranking-end-to-end.js +479 -0
- package/dist/bench/ranking-end-to-end.js.map +1 -0
- package/dist/bench/ranking-latency.d.ts +71 -0
- package/dist/bench/ranking-latency.d.ts.map +1 -0
- package/dist/bench/ranking-latency.js +130 -0
- package/dist/bench/ranking-latency.js.map +1 -0
- package/dist/bench/ranking-matrix.d.ts +141 -0
- package/dist/bench/ranking-matrix.d.ts.map +1 -0
- package/dist/bench/ranking-matrix.js +526 -0
- package/dist/bench/ranking-matrix.js.map +1 -0
- package/dist/bench/ranking-review.d.ts +93 -0
- package/dist/bench/ranking-review.d.ts.map +1 -0
- package/dist/bench/ranking-review.js +249 -0
- package/dist/bench/ranking-review.js.map +1 -0
- package/dist/bench/ranking.d.ts +122 -0
- package/dist/bench/ranking.d.ts.map +1 -0
- package/dist/bench/ranking.js +547 -0
- package/dist/bench/ranking.js.map +1 -0
- package/dist/bench.d.ts +60 -0
- package/dist/bench.d.ts.map +1 -0
- package/dist/bench.js +167 -0
- package/dist/bench.js.map +1 -0
- package/dist/config/jsonc.d.ts +8 -0
- package/dist/config/jsonc.d.ts.map +1 -0
- package/dist/config/jsonc.js +85 -0
- package/dist/config/jsonc.js.map +1 -0
- package/dist/config/load.d.ts +43 -0
- package/dist/config/load.d.ts.map +1 -0
- package/dist/config/load.js +608 -0
- package/dist/config/load.js.map +1 -0
- package/dist/config/paths.d.ts +10 -0
- package/dist/config/paths.d.ts.map +1 -0
- package/dist/config/paths.js +38 -0
- package/dist/config/paths.js.map +1 -0
- package/dist/config/schema.d.ts +698 -0
- package/dist/config/schema.d.ts.map +1 -0
- package/dist/config/schema.js +394 -0
- package/dist/config/schema.js.map +1 -0
- package/dist/config/write-rules.d.ts +41 -0
- package/dist/config/write-rules.d.ts.map +1 -0
- package/dist/config/write-rules.js +225 -0
- package/dist/config/write-rules.js.map +1 -0
- package/dist/config/write-setup.d.ts +42 -0
- package/dist/config/write-setup.d.ts.map +1 -0
- package/dist/config/write-setup.js +358 -0
- package/dist/config/write-setup.js.map +1 -0
- package/dist/context.d.ts +64 -0
- package/dist/context.d.ts.map +1 -0
- package/dist/context.js +14 -0
- package/dist/context.js.map +1 -0
- package/dist/doctor.d.ts +115 -0
- package/dist/doctor.d.ts.map +1 -0
- package/dist/doctor.js +388 -0
- package/dist/doctor.js.map +1 -0
- package/dist/index/chunk.d.ts +62 -0
- package/dist/index/chunk.d.ts.map +1 -0
- package/dist/index/chunk.js +307 -0
- package/dist/index/chunk.js.map +1 -0
- package/dist/index/defer.d.ts +32 -0
- package/dist/index/defer.d.ts.map +1 -0
- package/dist/index/defer.js +66 -0
- package/dist/index/defer.js.map +1 -0
- package/dist/index/derive.d.ts +99 -0
- package/dist/index/derive.d.ts.map +1 -0
- package/dist/index/derive.js +378 -0
- package/dist/index/derive.js.map +1 -0
- package/dist/index/entity-resolution.d.ts +68 -0
- package/dist/index/entity-resolution.d.ts.map +1 -0
- package/dist/index/entity-resolution.js +305 -0
- package/dist/index/entity-resolution.js.map +1 -0
- package/dist/index/graph.d.ts +46 -0
- package/dist/index/graph.d.ts.map +1 -0
- package/dist/index/graph.js +618 -0
- package/dist/index/graph.js.map +1 -0
- package/dist/index/indexer.d.ts +301 -0
- package/dist/index/indexer.d.ts.map +1 -0
- package/dist/index/indexer.js +1398 -0
- package/dist/index/indexer.js.map +1 -0
- package/dist/index/revision-barrier.d.ts +30 -0
- package/dist/index/revision-barrier.d.ts.map +1 -0
- package/dist/index/revision-barrier.js +140 -0
- package/dist/index/revision-barrier.js.map +1 -0
- package/dist/index.d.ts +55 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +37 -0
- package/dist/index.js.map +1 -0
- package/dist/ingest/adoption-eligibility.d.ts +4 -0
- package/dist/ingest/adoption-eligibility.d.ts.map +1 -0
- package/dist/ingest/adoption-eligibility.js +18 -0
- package/dist/ingest/adoption-eligibility.js.map +1 -0
- package/dist/ingest/availability.d.ts +15 -0
- package/dist/ingest/availability.d.ts.map +1 -0
- package/dist/ingest/availability.js +37 -0
- package/dist/ingest/availability.js.map +1 -0
- package/dist/ingest/extract.d.ts +58 -0
- package/dist/ingest/extract.d.ts.map +1 -0
- package/dist/ingest/extract.js +255 -0
- package/dist/ingest/extract.js.map +1 -0
- package/dist/ingest/fetch.d.ts +35 -0
- package/dist/ingest/fetch.d.ts.map +1 -0
- package/dist/ingest/fetch.js +142 -0
- package/dist/ingest/fetch.js.map +1 -0
- package/dist/ingest/inbox.d.ts +53 -0
- package/dist/ingest/inbox.d.ts.map +1 -0
- package/dist/ingest/inbox.js +110 -0
- package/dist/ingest/inbox.js.map +1 -0
- package/dist/ingest/name.d.ts +72 -0
- package/dist/ingest/name.d.ts.map +1 -0
- package/dist/ingest/name.js +208 -0
- package/dist/ingest/name.js.map +1 -0
- package/dist/ingest/parts.d.ts +78 -0
- package/dist/ingest/parts.d.ts.map +1 -0
- package/dist/ingest/parts.js +52 -0
- package/dist/ingest/parts.js.map +1 -0
- package/dist/ingest/rendition.d.ts +75 -0
- package/dist/ingest/rendition.d.ts.map +1 -0
- package/dist/ingest/rendition.js +92 -0
- package/dist/ingest/rendition.js.map +1 -0
- package/dist/ingest/store.d.ts +66 -0
- package/dist/ingest/store.d.ts.map +1 -0
- package/dist/ingest/store.js +92 -0
- package/dist/ingest/store.js.map +1 -0
- package/dist/kb/folders.d.ts +37 -0
- package/dist/kb/folders.d.ts.map +1 -0
- package/dist/kb/folders.js +133 -0
- package/dist/kb/folders.js.map +1 -0
- package/dist/kb/frontmatter.d.ts +91 -0
- package/dist/kb/frontmatter.d.ts.map +1 -0
- package/dist/kb/frontmatter.js +274 -0
- package/dist/kb/frontmatter.js.map +1 -0
- package/dist/kb/line-facts.d.ts +29 -0
- package/dist/kb/line-facts.d.ts.map +1 -0
- package/dist/kb/line-facts.js +23 -0
- package/dist/kb/line-facts.js.map +1 -0
- package/dist/kb/page.d.ts +92 -0
- package/dist/kb/page.d.ts.map +1 -0
- package/dist/kb/page.js +243 -0
- package/dist/kb/page.js.map +1 -0
- package/dist/kb/scan.d.ts +38 -0
- package/dist/kb/scan.d.ts.map +1 -0
- package/dist/kb/scan.js +109 -0
- package/dist/kb/scan.js.map +1 -0
- package/dist/kb/words.d.ts +30 -0
- package/dist/kb/words.d.ts.map +1 -0
- package/dist/kb/words.js +107 -0
- package/dist/kb/words.js.map +1 -0
- package/dist/maintenance/adopt.d.ts +65 -0
- package/dist/maintenance/adopt.d.ts.map +1 -0
- package/dist/maintenance/adopt.js +238 -0
- package/dist/maintenance/adopt.js.map +1 -0
- package/dist/maintenance/budget.d.ts +55 -0
- package/dist/maintenance/budget.d.ts.map +1 -0
- package/dist/maintenance/budget.js +61 -0
- package/dist/maintenance/budget.js.map +1 -0
- package/dist/maintenance/conflicts.d.ts +105 -0
- package/dist/maintenance/conflicts.d.ts.map +1 -0
- package/dist/maintenance/conflicts.js +368 -0
- package/dist/maintenance/conflicts.js.map +1 -0
- package/dist/maintenance/contradictions.d.ts +34 -0
- package/dist/maintenance/contradictions.d.ts.map +1 -0
- package/dist/maintenance/contradictions.js +199 -0
- package/dist/maintenance/contradictions.js.map +1 -0
- package/dist/maintenance/curate.d.ts +156 -0
- package/dist/maintenance/curate.d.ts.map +1 -0
- package/dist/maintenance/curate.js +2070 -0
- package/dist/maintenance/curate.js.map +1 -0
- package/dist/maintenance/dream.d.ts +139 -0
- package/dist/maintenance/dream.d.ts.map +1 -0
- package/dist/maintenance/dream.js +1632 -0
- package/dist/maintenance/dream.js.map +1 -0
- package/dist/maintenance/graph-candidates.d.ts +19 -0
- package/dist/maintenance/graph-candidates.d.ts.map +1 -0
- package/dist/maintenance/graph-candidates.js +120 -0
- package/dist/maintenance/graph-candidates.js.map +1 -0
- package/dist/maintenance/housekeeping.d.ts +81 -0
- package/dist/maintenance/housekeeping.d.ts.map +1 -0
- package/dist/maintenance/housekeeping.js +281 -0
- package/dist/maintenance/housekeeping.js.map +1 -0
- package/dist/maintenance/link-repairs.d.ts +58 -0
- package/dist/maintenance/link-repairs.d.ts.map +1 -0
- package/dist/maintenance/link-repairs.js +281 -0
- package/dist/maintenance/link-repairs.js.map +1 -0
- package/dist/maintenance/log.d.ts +41 -0
- package/dist/maintenance/log.d.ts.map +1 -0
- package/dist/maintenance/log.js +72 -0
- package/dist/maintenance/log.js.map +1 -0
- package/dist/maintenance/managed-item-routing.d.ts +48 -0
- package/dist/maintenance/managed-item-routing.d.ts.map +1 -0
- package/dist/maintenance/managed-item-routing.js +281 -0
- package/dist/maintenance/managed-item-routing.js.map +1 -0
- package/dist/maintenance/managed-item-sources.d.ts +38 -0
- package/dist/maintenance/managed-item-sources.d.ts.map +1 -0
- package/dist/maintenance/managed-item-sources.js +324 -0
- package/dist/maintenance/managed-item-sources.js.map +1 -0
- package/dist/maintenance/managed-items.d.ts +162 -0
- package/dist/maintenance/managed-items.d.ts.map +1 -0
- package/dist/maintenance/managed-items.js +1329 -0
- package/dist/maintenance/managed-items.js.map +1 -0
- package/dist/maintenance/merge-classifier.d.ts +19 -0
- package/dist/maintenance/merge-classifier.d.ts.map +1 -0
- package/dist/maintenance/merge-classifier.js +49 -0
- package/dist/maintenance/merge-classifier.js.map +1 -0
- package/dist/maintenance/model-telemetry.d.ts +45 -0
- package/dist/maintenance/model-telemetry.d.ts.map +1 -0
- package/dist/maintenance/model-telemetry.js +107 -0
- package/dist/maintenance/model-telemetry.js.map +1 -0
- package/dist/maintenance/observe.d.ts +72 -0
- package/dist/maintenance/observe.d.ts.map +1 -0
- package/dist/maintenance/observe.js +265 -0
- package/dist/maintenance/observe.js.map +1 -0
- package/dist/maintenance/path-policy.d.ts +83 -0
- package/dist/maintenance/path-policy.d.ts.map +1 -0
- package/dist/maintenance/path-policy.js +360 -0
- package/dist/maintenance/path-policy.js.map +1 -0
- package/dist/maintenance/plans.d.ts +366 -0
- package/dist/maintenance/plans.d.ts.map +1 -0
- package/dist/maintenance/plans.js +4060 -0
- package/dist/maintenance/plans.js.map +1 -0
- package/dist/maintenance/profile.d.ts +33 -0
- package/dist/maintenance/profile.d.ts.map +1 -0
- package/dist/maintenance/profile.js +99 -0
- package/dist/maintenance/profile.js.map +1 -0
- package/dist/maintenance/recovery.d.ts +38 -0
- package/dist/maintenance/recovery.d.ts.map +1 -0
- package/dist/maintenance/recovery.js +217 -0
- package/dist/maintenance/recovery.js.map +1 -0
- package/dist/maintenance/repair.d.ts +37 -0
- package/dist/maintenance/repair.d.ts.map +1 -0
- package/dist/maintenance/repair.js +183 -0
- package/dist/maintenance/repair.js.map +1 -0
- package/dist/maintenance/rule-drift.d.ts +104 -0
- package/dist/maintenance/rule-drift.d.ts.map +1 -0
- package/dist/maintenance/rule-drift.js +507 -0
- package/dist/maintenance/rule-drift.js.map +1 -0
- package/dist/maintenance/run-verification.d.ts +44 -0
- package/dist/maintenance/run-verification.d.ts.map +1 -0
- package/dist/maintenance/run-verification.js +253 -0
- package/dist/maintenance/run-verification.js.map +1 -0
- package/dist/maintenance/runs.d.ts +142 -0
- package/dist/maintenance/runs.d.ts.map +1 -0
- package/dist/maintenance/runs.js +439 -0
- package/dist/maintenance/runs.js.map +1 -0
- package/dist/maintenance/semantic-merge-discovery.d.ts +54 -0
- package/dist/maintenance/semantic-merge-discovery.d.ts.map +1 -0
- package/dist/maintenance/semantic-merge-discovery.js +308 -0
- package/dist/maintenance/semantic-merge-discovery.js.map +1 -0
- package/dist/maintenance/temporal.d.ts +44 -0
- package/dist/maintenance/temporal.d.ts.map +1 -0
- package/dist/maintenance/temporal.js +295 -0
- package/dist/maintenance/temporal.js.map +1 -0
- package/dist/models/client.d.ts +214 -0
- package/dist/models/client.d.ts.map +1 -0
- package/dist/models/client.js +904 -0
- package/dist/models/client.js.map +1 -0
- package/dist/models/provider-api.d.ts +28 -0
- package/dist/models/provider-api.d.ts.map +1 -0
- package/dist/models/provider-api.js +320 -0
- package/dist/models/provider-api.js.map +1 -0
- package/dist/open.d.ts +163 -0
- package/dist/open.d.ts.map +1 -0
- package/dist/open.js +382 -0
- package/dist/open.js.map +1 -0
- package/dist/ops/adopt.d.ts +12 -0
- package/dist/ops/adopt.d.ts.map +1 -0
- package/dist/ops/adopt.js +138 -0
- package/dist/ops/adopt.js.map +1 -0
- package/dist/ops/answer.d.ts +29 -0
- package/dist/ops/answer.d.ts.map +1 -0
- package/dist/ops/answer.js +557 -0
- package/dist/ops/answer.js.map +1 -0
- package/dist/ops/context.d.ts +18 -0
- package/dist/ops/context.d.ts.map +1 -0
- package/dist/ops/context.js +713 -0
- package/dist/ops/context.js.map +1 -0
- package/dist/ops/folder.d.ts +23 -0
- package/dist/ops/folder.d.ts.map +1 -0
- package/dist/ops/folder.js +130 -0
- package/dist/ops/folder.js.map +1 -0
- package/dist/ops/forget.d.ts +14 -0
- package/dist/ops/forget.d.ts.map +1 -0
- package/dist/ops/forget.js +212 -0
- package/dist/ops/forget.js.map +1 -0
- package/dist/ops/graph.d.ts +10 -0
- package/dist/ops/graph.d.ts.map +1 -0
- package/dist/ops/graph.js +480 -0
- package/dist/ops/graph.js.map +1 -0
- package/dist/ops/ingest.d.ts +40 -0
- package/dist/ops/ingest.d.ts.map +1 -0
- package/dist/ops/ingest.js +434 -0
- package/dist/ops/ingest.js.map +1 -0
- package/dist/ops/list.d.ts +9 -0
- package/dist/ops/list.d.ts.map +1 -0
- package/dist/ops/list.js +217 -0
- package/dist/ops/list.js.map +1 -0
- package/dist/ops/move.d.ts +20 -0
- package/dist/ops/move.d.ts.map +1 -0
- package/dist/ops/move.js +173 -0
- package/dist/ops/move.js.map +1 -0
- package/dist/ops/read.d.ts +9 -0
- package/dist/ops/read.d.ts.map +1 -0
- package/dist/ops/read.js +271 -0
- package/dist/ops/read.js.map +1 -0
- package/dist/ops/recall.d.ts +13 -0
- package/dist/ops/recall.d.ts.map +1 -0
- package/dist/ops/recall.js +297 -0
- package/dist/ops/recall.js.map +1 -0
- package/dist/ops/remember.d.ts +34 -0
- package/dist/ops/remember.d.ts.map +1 -0
- package/dist/ops/remember.js +660 -0
- package/dist/ops/remember.js.map +1 -0
- package/dist/ops/timeline.d.ts +11 -0
- package/dist/ops/timeline.d.ts.map +1 -0
- package/dist/ops/timeline.js +111 -0
- package/dist/ops/timeline.js.map +1 -0
- package/dist/ops/undo.d.ts +9 -0
- package/dist/ops/undo.d.ts.map +1 -0
- package/dist/ops/undo.js +38 -0
- package/dist/ops/undo.js.map +1 -0
- package/dist/ops/write.d.ts +55 -0
- package/dist/ops/write.d.ts.map +1 -0
- package/dist/ops/write.js +463 -0
- package/dist/ops/write.js.map +1 -0
- package/dist/recall/assemble.d.ts +81 -0
- package/dist/recall/assemble.d.ts.map +1 -0
- package/dist/recall/assemble.js +603 -0
- package/dist/recall/assemble.js.map +1 -0
- package/dist/recall/expand.d.ts +55 -0
- package/dist/recall/expand.d.ts.map +1 -0
- package/dist/recall/expand.js +282 -0
- package/dist/recall/expand.js.map +1 -0
- package/dist/recall/graph-arm.d.ts +17 -0
- package/dist/recall/graph-arm.d.ts.map +1 -0
- package/dist/recall/graph-arm.js +227 -0
- package/dist/recall/graph-arm.js.map +1 -0
- package/dist/recall/llm-rerank.d.ts +55 -0
- package/dist/recall/llm-rerank.d.ts.map +1 -0
- package/dist/recall/llm-rerank.js +213 -0
- package/dist/recall/llm-rerank.js.map +1 -0
- package/dist/recall/reranker-calibration.d.ts +23 -0
- package/dist/recall/reranker-calibration.d.ts.map +1 -0
- package/dist/recall/reranker-calibration.js +161 -0
- package/dist/recall/reranker-calibration.js.map +1 -0
- package/dist/recall/search.d.ts +107 -0
- package/dist/recall/search.d.ts.map +1 -0
- package/dist/recall/search.js +530 -0
- package/dist/recall/search.js.map +1 -0
- package/dist/reserved.d.ts +39 -0
- package/dist/reserved.d.ts.map +1 -0
- package/dist/reserved.js +63 -0
- package/dist/reserved.js.map +1 -0
- package/dist/rules/compile.d.ts +27 -0
- package/dist/rules/compile.d.ts.map +1 -0
- package/dist/rules/compile.js +105 -0
- package/dist/rules/compile.js.map +1 -0
- package/dist/setup/model-free.d.ts +15 -0
- package/dist/setup/model-free.d.ts.map +1 -0
- package/dist/setup/model-free.js +26 -0
- package/dist/setup/model-free.js.map +1 -0
- package/dist/setup/openai.d.ts +46 -0
- package/dist/setup/openai.d.ts.map +1 -0
- package/dist/setup/openai.js +165 -0
- package/dist/setup/openai.js.map +1 -0
- package/dist/store/db.d.ts +61 -0
- package/dist/store/db.d.ts.map +1 -0
- package/dist/store/db.js +265 -0
- package/dist/store/db.js.map +1 -0
- package/dist/store/ids.d.ts +29 -0
- package/dist/store/ids.d.ts.map +1 -0
- package/dist/store/ids.js +0 -0
- package/dist/store/ids.js.map +1 -0
- package/dist/store/migrations.d.ts +54 -0
- package/dist/store/migrations.d.ts.map +1 -0
- package/dist/store/migrations.js +923 -0
- package/dist/store/migrations.js.map +1 -0
- package/dist/store/vectors.d.ts +34 -0
- package/dist/store/vectors.d.ts.map +1 -0
- package/dist/store/vectors.js +158 -0
- package/dist/store/vectors.js.map +1 -0
- package/dist/timeline/documents.d.ts +23 -0
- package/dist/timeline/documents.d.ts.map +1 -0
- package/dist/timeline/documents.js +262 -0
- package/dist/timeline/documents.js.map +1 -0
- package/dist/watch/watcher.d.ts +43 -0
- package/dist/watch/watcher.d.ts.map +1 -0
- package/dist/watch/watcher.js +143 -0
- package/dist/watch/watcher.js.map +1 -0
- package/dist/write/atomic.d.ts +34 -0
- package/dist/write/atomic.d.ts.map +1 -0
- package/dist/write/atomic.js +58 -0
- package/dist/write/atomic.js.map +1 -0
- package/dist/write/conflict.d.ts +43 -0
- package/dist/write/conflict.d.ts.map +1 -0
- package/dist/write/conflict.js +149 -0
- package/dist/write/conflict.js.map +1 -0
- package/dist/write/edit.d.ts +59 -0
- package/dist/write/edit.d.ts.map +1 -0
- package/dist/write/edit.js +231 -0
- package/dist/write/edit.js.map +1 -0
- package/dist/write/gate.d.ts +59 -0
- package/dist/write/gate.d.ts.map +1 -0
- package/dist/write/gate.js +99 -0
- package/dist/write/gate.js.map +1 -0
- package/dist/write/journal.d.ts +90 -0
- package/dist/write/journal.d.ts.map +1 -0
- package/dist/write/journal.js +187 -0
- package/dist/write/journal.js.map +1 -0
- package/dist/write/ledger.d.ts +28 -0
- package/dist/write/ledger.d.ts.map +1 -0
- package/dist/write/ledger.js +141 -0
- package/dist/write/ledger.js.map +1 -0
- package/dist/write/placement.d.ts +29 -0
- package/dist/write/placement.d.ts.map +1 -0
- package/dist/write/placement.js +140 -0
- package/dist/write/placement.js.map +1 -0
- package/dist/write/remember-fallback.d.ts +18 -0
- package/dist/write/remember-fallback.d.ts.map +1 -0
- package/dist/write/remember-fallback.js +48 -0
- package/dist/write/remember-fallback.js.map +1 -0
- package/dist/write/retain.d.ts +61 -0
- package/dist/write/retain.d.ts.map +1 -0
- package/dist/write/retain.js +272 -0
- package/dist/write/retain.js.map +1 -0
- package/package.json +50 -0
- package/swift/extract.swift +232 -0
|
@@ -0,0 +1,2070 @@
|
|
|
1
|
+
import fsp from 'node:fs/promises';
|
|
2
|
+
import path from 'node:path';
|
|
3
|
+
import { z } from 'zod';
|
|
4
|
+
import { folderCatalog } from "../kb/folders.js";
|
|
5
|
+
import { parseFrontmatter, withAknoAliases } from "../kb/frontmatter.js";
|
|
6
|
+
import { AKNO_ITEM, normalizeLinkTarget, parsePage } from "../kb/page.js";
|
|
7
|
+
import { parseJsonLoose } from "../models/client.js";
|
|
8
|
+
import { effectiveRule, matchesGlob } from "../rules/compile.js";
|
|
9
|
+
import { missingNumericValues } from "./repair.js";
|
|
10
|
+
import { mergePathAllowed, pageAllowsMaintenanceTransform } from "./path-policy.js";
|
|
11
|
+
import { writeFileAtomic } from "../write/atomic.js";
|
|
12
|
+
import { fileEntry } from "../write/journal.js";
|
|
13
|
+
import { sha256 } from "../store/ids.js";
|
|
14
|
+
import { SEMANTIC_MERGE_PROMPT_VERSION } from "./merge-classifier.js";
|
|
15
|
+
import { discoverSemanticMergeCandidates, semanticMergePairKey, } from "./semantic-merge-discovery.js";
|
|
16
|
+
import { cleanTemporalProposal, inferTemporalMetadata, readTemporalDeclaration, temporalBoundaryCandidates, temporalClock, temporalPrompt, temporalState, withTemporalMetadata, } from "./temporal.js";
|
|
17
|
+
const HYGIENE_SYSTEM = `You are a conservative Markdown page hygienist. Reply with JSON only:
|
|
18
|
+
{"body":"the complete revised Markdown body"}
|
|
19
|
+
|
|
20
|
+
You may fix formatting, Markdown, grammar, awkward language and minor local organization. Preserve
|
|
21
|
+
the page's meaning and semantically equivalent top-level structure. Do not add facts. Do not remove
|
|
22
|
+
anything except exact duplicates. Keep every <!-- akno:item ... --> marker immediately before the
|
|
23
|
+
knowledge it identifies. Do not add frontmatter.`;
|
|
24
|
+
export const HYGIENE_SCHEMA = z.object({ body: z.string() });
|
|
25
|
+
const SYNTHESIZE_SYSTEM = `You synthesize one canonical Markdown knowledge page from its current body
|
|
26
|
+
and linked evidence. Reply with JSON only:
|
|
27
|
+
{"body":"complete canonical Markdown body","splits":[{"suffix":"topic","title":"Title","body":"complete child body"}],"extracts":[{"slug":"allowed-folder/topic","title":"Title","source_heading":"## Exact heading from current body","bridge":"See [[allowed-folder/topic]]."}],"temporal":false}
|
|
28
|
+
|
|
29
|
+
You may fully rewrite and restructure the body. Accumulate knowledge by subject; link to evidence and
|
|
30
|
+
related pages in the sections they support instead of repeating whole source pages. A link or backlink
|
|
31
|
+
is only a relevance hint: use a linked fact only when it is directly about this canonical subject.
|
|
32
|
+
Do not copy cross-cutting trip, passport, accommodation, booking or itinerary boilerplate into every
|
|
33
|
+
place page. When such a page is genuinely useful, link it once without restating its general details.
|
|
34
|
+
Internal page links must use [[the/exact-supplied-slug]]. Never invent a URL, relative path or slug,
|
|
35
|
+
and never alter or remove an existing link target. Reorganize rather
|
|
36
|
+
than summarize: preserve every factual detail already in the canonical body, including dates, times,
|
|
37
|
+
prices, measurements, descriptions, access instructions and practical guidance. Numeric formatting and
|
|
38
|
+
sentence punctuation may change, but no value may disappear. Use ## Unresolved only when the supplied
|
|
39
|
+
conflict list contains a real unresolved conflict; do not manufacture one from compatible access rules
|
|
40
|
+
or descriptions of different areas. Do not choose a side without evidence. Keep every
|
|
41
|
+
<!-- akno:item ... --> marker exactly once, immediately before the knowledge it identifies. The
|
|
42
|
+
canonical page remains at its current slug. Suggest splits only for genuinely oversized, coherent
|
|
43
|
+
sections. Child suffixes are one lowercase hyphenated path segment. Do not add frontmatter.
|
|
44
|
+
|
|
45
|
+
An extraction is different from a split: move one coherent, reusable subject out while the source
|
|
46
|
+
retains its primary purpose. Propose at most one extraction, only into an exact allowed destination
|
|
47
|
+
folder supplied by the user message. Use a lowercase-hyphenated basename and never make the target a
|
|
48
|
+
child of the source page. Select one exact eligible Markdown heading supplied by the user message in
|
|
49
|
+
"source_heading". Akno—not you—will move that complete section verbatim, including its item markers,
|
|
50
|
+
provenance and links, and will insert the bridge at the same boundary. When proposing an extraction,
|
|
51
|
+
reproduce the complete current body byte for byte in "body"; do not remove, copy, summarize or rewrite
|
|
52
|
+
the selected section yourself. The short bridge must link to the exact proposed slug. Do not propose a
|
|
53
|
+
split and an extraction for the same page.
|
|
54
|
+
|
|
55
|
+
Always send "splits" and "extracts" — empty arrays when there is nothing to move.
|
|
56
|
+
|
|
57
|
+
When no temporal boundary is supplied but the user message lists explicit boundary candidates, set
|
|
58
|
+
"temporal" to {"kind":"event","start":"date or timestamp","until":"date or timestamp","timezone":"IANA zone"}
|
|
59
|
+
only when this whole page is a bounded event. Use only listed dates, use null for unknown fields and
|
|
60
|
+
false for evergreen or ambiguous pages. A date means the complete local day; never invent an
|
|
61
|
+
end-of-day time.`;
|
|
62
|
+
/**
|
|
63
|
+
* Every field is required and `false`/`null` carry the "nothing here" cases, because strict
|
|
64
|
+
* mode rejects an optional property — so the prompt above was reworded to match. Neither is a
|
|
65
|
+
* behaviour change: `cleanSplits` already treats `[]` exactly as it treats an absent list, and
|
|
66
|
+
* `cleanTemporal` already reads a null `start` or `timezone` as unset.
|
|
67
|
+
*
|
|
68
|
+
* The boundary this schema cannot enforce is the one that matters most: `cleanTemporalProposal`
|
|
69
|
+
* still rejects any date the page did not itself supply. A grammar can require a date-shaped
|
|
70
|
+
* string; only the caller knows which dates were on the page.
|
|
71
|
+
*/
|
|
72
|
+
export const SYNTHESIZE_SCHEMA = z.object({
|
|
73
|
+
body: z.string(),
|
|
74
|
+
splits: z.array(z.object({ suffix: z.string(), title: z.string(), body: z.string() })),
|
|
75
|
+
extracts: z.array(z.object({
|
|
76
|
+
slug: z.string(),
|
|
77
|
+
title: z.string(),
|
|
78
|
+
source_heading: z.string(),
|
|
79
|
+
bridge: z.string(),
|
|
80
|
+
})),
|
|
81
|
+
temporal: z.union([
|
|
82
|
+
z.literal(false),
|
|
83
|
+
z.object({
|
|
84
|
+
kind: z.literal('event'),
|
|
85
|
+
start: z.string().nullable(),
|
|
86
|
+
until: z.string(),
|
|
87
|
+
timezone: z.string().nullable(),
|
|
88
|
+
}),
|
|
89
|
+
]),
|
|
90
|
+
});
|
|
91
|
+
const MERGE_SYSTEM = `You merge two Markdown pages that a guarded candidate source identifies as the same
|
|
92
|
+
durable subject. The source is an explicit exact alias, multiple exact current graph-resolved attributes, or a
|
|
93
|
+
qualified semantic prefilter plus a strict same-subject classifier. Candidate discovery is not write authority.
|
|
94
|
+
Reply with JSON only:
|
|
95
|
+
{"body":"complete merged canonical Markdown body"}
|
|
96
|
+
|
|
97
|
+
The user message supplies a canonical body and a prepared duplicate body. Preserve every non-blank line from
|
|
98
|
+
both inputs verbatim. An exactly identical line repeated by both inputs may appear once, but otherwise do not
|
|
99
|
+
rewrite, summarize, combine, or omit lines. You may interleave complete sections to make one coherent page,
|
|
100
|
+
but preserve line order within each input. Do not add connective prose, headings, frontmatter, facts, links, or item markers. Keep each
|
|
101
|
+
<!-- akno:item ... --> marker immediately before the knowledge it identifies. The canonical page's first H1
|
|
102
|
+
must remain its first H1. The duplicate title is preserved separately as an alias, so its leading H1 may already
|
|
103
|
+
have been removed from the prepared duplicate body.`;
|
|
104
|
+
export const MERGE_SCHEMA = z.object({ body: z.string() });
|
|
105
|
+
const ARCHIVE_SYSTEM = `${SYNTHESIZE_SYSTEM}
|
|
106
|
+
|
|
107
|
+
This page describes an event that has ended. This is an archival synthesis, not another planning pass.
|
|
108
|
+
Integrate supported outcomes, later facts, direct links and resolutions, but never infer that a planned
|
|
109
|
+
activity happened merely because its date passed. Preserve plans as plans unless supplied evidence confirms
|
|
110
|
+
their outcome. Do not refresh operational advice or reorganize the page without a substantive archival gain.
|
|
111
|
+
If no meaningful post-event knowledge is supplied, reproduce the current body byte for byte.`;
|
|
112
|
+
const VERIFY_SYSTEM = `You verify an automatic Markdown rewrite. Reply with JSON only:
|
|
113
|
+
{"ok":true,"issues":[]}
|
|
114
|
+
|
|
115
|
+
Reject a hygiene rewrite if it changes meaning, loses non-duplicate knowledge, adds facts, or makes
|
|
116
|
+
more than minor structural changes. Reject a synthesis rewrite if it invents facts, loses supported
|
|
117
|
+
knowledge, hides a conflict, misattributes evidence, repeats unrelated cross-cutting logistics, changes
|
|
118
|
+
an existing link target, invents a URL, creates an incoherent split, or extracts content that is not a
|
|
119
|
+
coherent reusable subject. For an extraction, require every moved authored line to remain verbatim,
|
|
120
|
+
the source to retain its primary purpose, a useful source bridge, a source backlink, and an independent
|
|
121
|
+
destination rather than a disguised child split. A backlink is only a relevance
|
|
122
|
+
hint, not evidence that every fact on that page belongs here. Stable item markers are metadata, not
|
|
123
|
+
prose, and must remain attached to their knowledge. For an archival rewrite, reject any claim that
|
|
124
|
+
a plan happened merely because its date passed, and reject restructuring with no substantive
|
|
125
|
+
post-event knowledge.`;
|
|
126
|
+
const VERIFY_MERGE_SYSTEM = `${VERIFY_SYSTEM}
|
|
127
|
+
|
|
128
|
+
For a merge, require the supplied sealed candidate signal to establish one durable identity. Exact aliases are
|
|
129
|
+
direct evidence. Exact graph-resolved attributes or a qualified semantic classifier are candidate evidence,
|
|
130
|
+
not permission to merge a merely related or intentionally scoped page. Reject the merge if the two
|
|
131
|
+
pages merely concern related subjects, if their separate purposes remain useful, if any unique authored detail or provenance marker is lost, if an
|
|
132
|
+
unrelated page is rewritten, if an inbound link is not redirected, or if deleting the duplicate would orphan
|
|
133
|
+
owned evidence. Exact duplicate lines may be deduplicated.`;
|
|
134
|
+
export const VERIFY_SCHEMA = z.object({ ok: z.boolean(), issues: z.array(z.string()) });
|
|
135
|
+
// Changing a prompt or a deterministic rule must invalidate the decisions made by its predecessor.
|
|
136
|
+
// 13: qualified semantic candidates join exact/graph discovery without gaining write authority.
|
|
137
|
+
// Decisions from the previous transformation surface must be reconsidered once.
|
|
138
|
+
const CURATE_FINGERPRINT_VERSION = 13;
|
|
139
|
+
export async function curatePages(ctx, options) {
|
|
140
|
+
const settings = ctx.config.maintenance.curate;
|
|
141
|
+
const allowedKinds = options.allowedKinds ??
|
|
142
|
+
new Set(['hygiene', 'synthesis', 'split', 'extract', 'merge']);
|
|
143
|
+
const result = {
|
|
144
|
+
pages: [],
|
|
145
|
+
files: [],
|
|
146
|
+
changeId: null,
|
|
147
|
+
warnings: [],
|
|
148
|
+
degraded: [],
|
|
149
|
+
semanticMerge: null,
|
|
150
|
+
drafts: [],
|
|
151
|
+
};
|
|
152
|
+
const rows = ctx.store.db
|
|
153
|
+
.prepare(`SELECT id, slug, rel_path, title, role, dream_management, about, frontmatter, aliases, body_hash, bytes,
|
|
154
|
+
curate_input_hash, curate_status
|
|
155
|
+
FROM pages
|
|
156
|
+
WHERE dream_management IN ('hygiene', 'synthesize') AND role = 'knowledge'
|
|
157
|
+
ORDER BY updated_at DESC, slug`)
|
|
158
|
+
.all();
|
|
159
|
+
const knownSlugs = new Set(ctx.store.db.prepare('SELECT slug FROM pages').all().map((row) => row.slug.toLowerCase()));
|
|
160
|
+
const extractionFolders = allowedExtractionFolders(ctx);
|
|
161
|
+
const extractionPolicyHash = extractionPolicyFingerprint(ctx, extractionFolders);
|
|
162
|
+
const clock = temporalClock();
|
|
163
|
+
const splitLimit = allowedKinds.has('split') ? settings.maxSplits : 0;
|
|
164
|
+
const extractLimit = allowedKinds.has('extract') ? settings.maxExtracts : 0;
|
|
165
|
+
let splitBudget = splitLimit;
|
|
166
|
+
let extractBudget = extractLimit;
|
|
167
|
+
let attempted = 0;
|
|
168
|
+
const state = new Map();
|
|
169
|
+
const staged = [];
|
|
170
|
+
const mergeDrafts = [];
|
|
171
|
+
const mergeReserved = new Set();
|
|
172
|
+
const mergeOperationPaths = new Set();
|
|
173
|
+
// Merge is available only through durable plans. The legacy `write` switch cannot represent
|
|
174
|
+
// a separately decided deletion, while audit/review/auto all seal the exact multi-file item.
|
|
175
|
+
if (options.includePreviewed && allowedKinds.has('merge') && settings.maxMerges > 0) {
|
|
176
|
+
const discovery = await discoverMergeCandidates(ctx, rows, settings.mergeFolders);
|
|
177
|
+
const candidates = discovery.candidates;
|
|
178
|
+
result.warnings.push(...discovery.warnings);
|
|
179
|
+
result.degraded.push(...discovery.degraded);
|
|
180
|
+
result.semanticMerge = discovery.semanticMerge;
|
|
181
|
+
for (const candidate of candidates) {
|
|
182
|
+
mergeReserved.add(candidate.canonical.id);
|
|
183
|
+
mergeReserved.add(candidate.duplicate.id);
|
|
184
|
+
}
|
|
185
|
+
let mergeAttempts = 0;
|
|
186
|
+
for (const candidate of candidates) {
|
|
187
|
+
if (mergeAttempts >= settings.maxMerges)
|
|
188
|
+
break;
|
|
189
|
+
if (attempted + 2 > settings.maxPages)
|
|
190
|
+
break;
|
|
191
|
+
const inspection = await inspectMergeCandidate(ctx, candidate);
|
|
192
|
+
if (!inspection) {
|
|
193
|
+
result.warnings.push(`${candidate.canonical.slug}: could not read the ${candidate.identityKind} merge candidate ${candidate.duplicate.slug}`);
|
|
194
|
+
continue;
|
|
195
|
+
}
|
|
196
|
+
for (const page of inspection.inbound)
|
|
197
|
+
mergeReserved.add(page.id);
|
|
198
|
+
if (!curationDue(candidate.canonical, inspection.inputHash, options.dryRun, true))
|
|
199
|
+
continue;
|
|
200
|
+
mergeAttempts++;
|
|
201
|
+
attempted += 2;
|
|
202
|
+
const prepared = await prepareMergeDraft(ctx, inspection);
|
|
203
|
+
const paths = prepared.draft
|
|
204
|
+
? [
|
|
205
|
+
prepared.draft.relPath,
|
|
206
|
+
...prepared.draft.merge.linkUpdates.map((update) => update.relPath),
|
|
207
|
+
prepared.draft.merge.sourceRelPath,
|
|
208
|
+
]
|
|
209
|
+
: [];
|
|
210
|
+
if (paths.some((relPath) => mergeOperationPaths.has(relPath))) {
|
|
211
|
+
prepared.draft = null;
|
|
212
|
+
prepared.issues = ['merge overlaps another planned merge operation in this run'];
|
|
213
|
+
prepared.cacheable = true;
|
|
214
|
+
}
|
|
215
|
+
if (prepared.issues.length > 0 || !prepared.draft) {
|
|
216
|
+
result.pages.push({
|
|
217
|
+
slug: candidate.canonical.slug,
|
|
218
|
+
mode: 'synthesize',
|
|
219
|
+
action: 'rejected',
|
|
220
|
+
splits: [],
|
|
221
|
+
extractions: [],
|
|
222
|
+
merges: [candidate.duplicate.slug],
|
|
223
|
+
issues: prepared.issues.length > 0 ? prepared.issues : ['merge planner returned no exact draft'],
|
|
224
|
+
});
|
|
225
|
+
if (prepared.cacheable) {
|
|
226
|
+
queueCurateState(state, candidate.canonical.id, prepared.inputHash, 'rejected');
|
|
227
|
+
}
|
|
228
|
+
continue;
|
|
229
|
+
}
|
|
230
|
+
mergeDrafts.push(prepared.draft);
|
|
231
|
+
for (const relPath of paths)
|
|
232
|
+
mergeOperationPaths.add(relPath);
|
|
233
|
+
for (const update of prepared.draft.merge.linkUpdates) {
|
|
234
|
+
const row = rows.find((page) => page.slug === update.slug);
|
|
235
|
+
if (row)
|
|
236
|
+
mergeReserved.add(row.id);
|
|
237
|
+
}
|
|
238
|
+
result.pages.push({
|
|
239
|
+
slug: candidate.canonical.slug,
|
|
240
|
+
mode: 'synthesize',
|
|
241
|
+
action: 'would-update',
|
|
242
|
+
splits: [],
|
|
243
|
+
extractions: [],
|
|
244
|
+
merges: [candidate.duplicate.slug],
|
|
245
|
+
issues: [],
|
|
246
|
+
});
|
|
247
|
+
}
|
|
248
|
+
}
|
|
249
|
+
for (const row of rows) {
|
|
250
|
+
if (mergeReserved.has(row.id))
|
|
251
|
+
continue;
|
|
252
|
+
const pathKind = row.dream_management === 'hygiene' ? 'hygiene' : 'synthesis';
|
|
253
|
+
if (!pageAllowsMaintenanceTransform(ctx.config, { slug: row.slug, role: row.role, dreamManagement: row.dream_management }, pathKind)) {
|
|
254
|
+
continue;
|
|
255
|
+
}
|
|
256
|
+
if (row.dream_management === 'hygiene' && !allowedKinds.has('hygiene'))
|
|
257
|
+
continue;
|
|
258
|
+
if (row.dream_management === 'synthesize' &&
|
|
259
|
+
!allowedKinds.has('synthesis') &&
|
|
260
|
+
!allowedKinds.has('split') &&
|
|
261
|
+
!allowedKinds.has('extract')) {
|
|
262
|
+
continue;
|
|
263
|
+
}
|
|
264
|
+
const before = await fsp.readFile(path.join(ctx.config.aknoPath, row.rel_path), 'utf8').catch(() => null);
|
|
265
|
+
if (before === null) {
|
|
266
|
+
result.warnings.push(`${row.slug}: could not read page`);
|
|
267
|
+
continue;
|
|
268
|
+
}
|
|
269
|
+
const fm = parseFrontmatter(before);
|
|
270
|
+
const body = before.slice(fm.bodyOffset);
|
|
271
|
+
const inferenceInput = { slug: row.slug, title: row.title, frontmatter: fm.data, body };
|
|
272
|
+
const declaration = readTemporalDeclaration(fm.data);
|
|
273
|
+
if (declaration.invalid) {
|
|
274
|
+
result.warnings.push(`${row.slug}: akno.temporal is malformed; automatic temporal handling was skipped`);
|
|
275
|
+
}
|
|
276
|
+
let temporal = declaration.metadata;
|
|
277
|
+
let temporalSource = temporal ? 'declared' : null;
|
|
278
|
+
if (!temporal && !declaration.disabled && !declaration.invalid && row.dream_management === 'synthesize') {
|
|
279
|
+
temporal = inferTemporalMetadata(inferenceInput);
|
|
280
|
+
if (temporal)
|
|
281
|
+
temporalSource = 'inferred';
|
|
282
|
+
}
|
|
283
|
+
let eventState = temporal ? temporalState(temporal, clock) : null;
|
|
284
|
+
let archival = row.dream_management === 'synthesize' && eventState === 'past';
|
|
285
|
+
const allEvidence = row.dream_management === 'synthesize' ? evidenceFor(ctx, row) : [];
|
|
286
|
+
let evidence = archival ? archivalEvidence(allEvidence) : allEvidence;
|
|
287
|
+
const conflicts = row.dream_management === 'synthesize' ? conflictsFor(ctx, row.id) : [];
|
|
288
|
+
let inputHash = curateInputHash(row, evidence, conflicts, temporal, eventState, extractionPolicyHash, incomingLinkFingerprint(ctx, row.id));
|
|
289
|
+
if (!curationDue(row, inputHash, options.dryRun, options.includePreviewed ?? false))
|
|
290
|
+
continue;
|
|
291
|
+
if (attempted >= settings.maxPages)
|
|
292
|
+
break;
|
|
293
|
+
attempted++;
|
|
294
|
+
const candidates = row.dream_management === 'synthesize' && !temporal && !declaration.disabled && !declaration.invalid
|
|
295
|
+
? temporalBoundaryCandidates(inferenceInput)
|
|
296
|
+
: [];
|
|
297
|
+
const prompt = row.dream_management === 'hygiene' ? HYGIENE_SYSTEM : archival ? ARCHIVE_SYSTEM : SYNTHESIZE_SYSTEM;
|
|
298
|
+
const sourceSections = extractionSections(body, settings.extractSectionBytes);
|
|
299
|
+
const canRequestExtraction = row.dream_management === 'synthesize' &&
|
|
300
|
+
!archival &&
|
|
301
|
+
extractBudget > 0 &&
|
|
302
|
+
Buffer.byteLength(before) >= settings.extractAfterBytes &&
|
|
303
|
+
extractionFolders.length > 0 &&
|
|
304
|
+
sourceSections.length > 0;
|
|
305
|
+
const draftResult = await ctx.models.derive.chat([
|
|
306
|
+
{ role: 'system', content: prompt },
|
|
307
|
+
{
|
|
308
|
+
role: 'user',
|
|
309
|
+
content: `${temporalPrompt(temporal, clock)}\n\nSlug: ${row.slug}\nTitle: ${row.title}` +
|
|
310
|
+
(row.dream_management === 'synthesize'
|
|
311
|
+
? extractionPrompt(canRequestExtraction ? extractionFolders : [], canRequestExtraction ? sourceSections : [])
|
|
312
|
+
: '') +
|
|
313
|
+
(candidates.length
|
|
314
|
+
? `\nTemporal boundary candidates explicitly present in this page: ${candidates.join(', ')}`
|
|
315
|
+
: '') +
|
|
316
|
+
`\n\nCurrent body:\n${body.slice(0, 40_000)}` +
|
|
317
|
+
(evidence.length
|
|
318
|
+
? `\n\nEvidence graph:\n${renderEvidence(evidence).join('\n\n').slice(0, 40_000)}`
|
|
319
|
+
: '') +
|
|
320
|
+
(conflicts.length ? `\n\nUnresolved conflicts:\n${renderConflicts(conflicts).join('\n')}` : ''),
|
|
321
|
+
},
|
|
322
|
+
], {
|
|
323
|
+
schema: row.dream_management === 'hygiene' ? HYGIENE_SCHEMA : SYNTHESIZE_SCHEMA,
|
|
324
|
+
maxTokens: 8_000,
|
|
325
|
+
});
|
|
326
|
+
const parsed = draftResult.ok && draftResult.value ? parseJsonLoose(draftResult.value) : null;
|
|
327
|
+
let nextBody = typeof parsed?.body === 'string' ? endWithNewline(parsed.body) : null;
|
|
328
|
+
if (!nextBody) {
|
|
329
|
+
if (draftResult.ok)
|
|
330
|
+
ctx.models.derive.reportInvalidResponse();
|
|
331
|
+
const issue = draftResult.error ?? 'draft was not valid JSON with a body';
|
|
332
|
+
result.pages.push({
|
|
333
|
+
slug: row.slug,
|
|
334
|
+
mode: row.dream_management,
|
|
335
|
+
action: 'rejected',
|
|
336
|
+
splits: [],
|
|
337
|
+
extractions: [],
|
|
338
|
+
merges: [],
|
|
339
|
+
issues: [issue],
|
|
340
|
+
...temporalResult(temporal, temporalSource, clock, archival),
|
|
341
|
+
});
|
|
342
|
+
// Provider/transport failures are retryable. A successful model call that returned an
|
|
343
|
+
// unusable draft is a completed rejection and should not burn another call next night.
|
|
344
|
+
if (draftResult.ok)
|
|
345
|
+
queueCurateState(state, row.id, inputHash, 'rejected');
|
|
346
|
+
continue;
|
|
347
|
+
}
|
|
348
|
+
let metadataOnly = false;
|
|
349
|
+
if (!temporal && row.dream_management === 'synthesize' && candidates.length > 0) {
|
|
350
|
+
const proposed = cleanTemporalProposal(parsed?.temporal, candidates);
|
|
351
|
+
if (proposed.issue) {
|
|
352
|
+
result.pages.push({
|
|
353
|
+
slug: row.slug,
|
|
354
|
+
mode: row.dream_management,
|
|
355
|
+
action: 'rejected',
|
|
356
|
+
splits: [],
|
|
357
|
+
extractions: [],
|
|
358
|
+
merges: [],
|
|
359
|
+
issues: [proposed.issue],
|
|
360
|
+
});
|
|
361
|
+
queueCurateState(state, row.id, inputHash, 'rejected');
|
|
362
|
+
continue;
|
|
363
|
+
}
|
|
364
|
+
if (proposed.metadata) {
|
|
365
|
+
temporal = proposed.metadata;
|
|
366
|
+
temporalSource = 'model';
|
|
367
|
+
eventState = temporalState(temporal, clock);
|
|
368
|
+
archival = eventState === 'past';
|
|
369
|
+
// The first call classified an unmarked page without the archival contract. Persist the
|
|
370
|
+
// boundary alone and let the next fingerprinted pass assess the ended event correctly.
|
|
371
|
+
if (archival) {
|
|
372
|
+
nextBody = body;
|
|
373
|
+
evidence = archivalEvidence(allEvidence);
|
|
374
|
+
inputHash = curateInputHash(row, evidence, conflicts, temporal, eventState, extractionPolicyHash, incomingLinkFingerprint(ctx, row.id));
|
|
375
|
+
metadataOnly = true;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
const temporalBase = temporal && temporalSource !== 'declared' ? withTemporalMetadata(before, temporal) : before;
|
|
380
|
+
if (temporalBase === null) {
|
|
381
|
+
const issue = 'could not add akno.temporal without reformatting existing frontmatter';
|
|
382
|
+
result.pages.push({
|
|
383
|
+
slug: row.slug,
|
|
384
|
+
mode: row.dream_management,
|
|
385
|
+
action: 'rejected',
|
|
386
|
+
splits: [],
|
|
387
|
+
extractions: [],
|
|
388
|
+
merges: [],
|
|
389
|
+
issues: [issue],
|
|
390
|
+
...temporalResult(temporal, temporalSource, clock, archival),
|
|
391
|
+
});
|
|
392
|
+
queueCurateState(state, row.id, inputHash, 'rejected');
|
|
393
|
+
continue;
|
|
394
|
+
}
|
|
395
|
+
const archivalNoop = archival && archiveMeaningKey(body) === archiveMeaningKey(nextBody);
|
|
396
|
+
if (archivalNoop)
|
|
397
|
+
nextBody = body;
|
|
398
|
+
const maySplit = !metadataOnly &&
|
|
399
|
+
!archivalNoop &&
|
|
400
|
+
row.dream_management === 'synthesize' &&
|
|
401
|
+
Buffer.byteLength(before) >= settings.splitAfterBytes;
|
|
402
|
+
const splits = maySplit
|
|
403
|
+
? cleanSplits(parsed?.splits, Math.min(settings.maxChildrenPerPage, splitBudget), settings.splitSectionBytes)
|
|
404
|
+
: [];
|
|
405
|
+
const extractionResult = cleanExtractions(parsed?.extracts, {
|
|
406
|
+
available: canRequestExtraction && !metadataOnly && !archivalNoop,
|
|
407
|
+
limit: Math.min(1, extractBudget),
|
|
408
|
+
minBytes: settings.extractSectionBytes,
|
|
409
|
+
sourceSlug: row.slug,
|
|
410
|
+
folders: extractionFolders,
|
|
411
|
+
knownSlugs,
|
|
412
|
+
rules: ctx.config.rules,
|
|
413
|
+
sourceBody: body,
|
|
414
|
+
});
|
|
415
|
+
const extractions = extractionResult.extractions;
|
|
416
|
+
if (extractionResult.issues.length > 0) {
|
|
417
|
+
result.pages.push({
|
|
418
|
+
slug: row.slug,
|
|
419
|
+
mode: row.dream_management,
|
|
420
|
+
action: 'rejected',
|
|
421
|
+
splits: [],
|
|
422
|
+
extractions: [],
|
|
423
|
+
merges: [],
|
|
424
|
+
issues: extractionResult.issues,
|
|
425
|
+
...temporalResult(temporal, temporalSource, clock, archival),
|
|
426
|
+
});
|
|
427
|
+
queueCurateState(state, row.id, inputHash, 'rejected');
|
|
428
|
+
continue;
|
|
429
|
+
}
|
|
430
|
+
if (extractions.length > 0)
|
|
431
|
+
nextBody = withExtractionBridge(body, extractions[0]);
|
|
432
|
+
const incomingAnchors = extractions.length > 0 ? await incomingHeadingAnchors(ctx, row.id, row.slug) : new Set();
|
|
433
|
+
const deterministic = guardRewrite({
|
|
434
|
+
mode: row.dream_management,
|
|
435
|
+
before: body,
|
|
436
|
+
after: nextBody,
|
|
437
|
+
splits,
|
|
438
|
+
extractions,
|
|
439
|
+
conflicts,
|
|
440
|
+
pageSlug: row.slug,
|
|
441
|
+
knownSlugs,
|
|
442
|
+
allowedLinkSlugs: new Set(evidence.map((entry) => entry.slug.toLowerCase())),
|
|
443
|
+
incomingAnchors,
|
|
444
|
+
});
|
|
445
|
+
if (deterministic.length > 0) {
|
|
446
|
+
result.pages.push({
|
|
447
|
+
slug: row.slug,
|
|
448
|
+
mode: row.dream_management,
|
|
449
|
+
action: 'rejected',
|
|
450
|
+
splits: [],
|
|
451
|
+
extractions: [],
|
|
452
|
+
merges: [],
|
|
453
|
+
issues: deterministic,
|
|
454
|
+
...temporalResult(temporal, temporalSource, clock, archival),
|
|
455
|
+
});
|
|
456
|
+
queueCurateState(state, row.id, inputHash, 'rejected');
|
|
457
|
+
continue;
|
|
458
|
+
}
|
|
459
|
+
const verified = metadataOnly || archivalNoop || (nextBody === body && splits.length === 0 && extractions.length === 0)
|
|
460
|
+
? { ok: true, issues: [], cacheable: true }
|
|
461
|
+
: await verifyDraft(ctx, row, body, nextBody, splits, extractions, evidence, conflicts, temporal, clock, archival);
|
|
462
|
+
if (!verified.ok) {
|
|
463
|
+
result.pages.push({
|
|
464
|
+
slug: row.slug,
|
|
465
|
+
mode: row.dream_management,
|
|
466
|
+
action: 'rejected',
|
|
467
|
+
splits: [],
|
|
468
|
+
extractions: [],
|
|
469
|
+
merges: [],
|
|
470
|
+
issues: verified.issues,
|
|
471
|
+
...temporalResult(temporal, temporalSource, clock, archival),
|
|
472
|
+
});
|
|
473
|
+
if (verified.cacheable)
|
|
474
|
+
queueCurateState(state, row.id, inputHash, 'rejected');
|
|
475
|
+
continue;
|
|
476
|
+
}
|
|
477
|
+
const children = splits.map((split) => {
|
|
478
|
+
const slug = `${row.slug}/${split.suffix}`;
|
|
479
|
+
return {
|
|
480
|
+
slug,
|
|
481
|
+
relPath: `${slug}.md`,
|
|
482
|
+
content: childPage(split, row.slug),
|
|
483
|
+
};
|
|
484
|
+
});
|
|
485
|
+
const extractedPages = extractions.map((extraction) => ({
|
|
486
|
+
slug: extraction.slug,
|
|
487
|
+
relPath: `${extraction.slug}.md`,
|
|
488
|
+
content: extractionPage(extraction, row.slug),
|
|
489
|
+
}));
|
|
490
|
+
const temporalFm = parseFrontmatter(temporalBase);
|
|
491
|
+
const after = temporalBase.slice(0, temporalFm.bodyOffset) + nextBody;
|
|
492
|
+
if (after === before && children.length === 0 && extractedPages.length === 0) {
|
|
493
|
+
result.pages.push({
|
|
494
|
+
slug: row.slug,
|
|
495
|
+
mode: row.dream_management,
|
|
496
|
+
action: 'unchanged',
|
|
497
|
+
splits: [],
|
|
498
|
+
extractions: [],
|
|
499
|
+
merges: [],
|
|
500
|
+
issues: [],
|
|
501
|
+
...temporalResult(temporal, temporalSource, clock, archival),
|
|
502
|
+
});
|
|
503
|
+
queueCurateState(state, row.id, inputHash, 'unchanged');
|
|
504
|
+
continue;
|
|
505
|
+
}
|
|
506
|
+
const transformationKind = row.dream_management === 'hygiene'
|
|
507
|
+
? 'hygiene'
|
|
508
|
+
: extractedPages.length > 0
|
|
509
|
+
? 'extract'
|
|
510
|
+
: children.length > 0
|
|
511
|
+
? 'split'
|
|
512
|
+
: 'synthesis';
|
|
513
|
+
if (!allowedKinds.has(transformationKind))
|
|
514
|
+
continue;
|
|
515
|
+
staged.push({
|
|
516
|
+
row,
|
|
517
|
+
before,
|
|
518
|
+
after,
|
|
519
|
+
children,
|
|
520
|
+
extractions: extractedPages,
|
|
521
|
+
evidence,
|
|
522
|
+
conflicts,
|
|
523
|
+
inputHash,
|
|
524
|
+
metadataOnly,
|
|
525
|
+
});
|
|
526
|
+
splitBudget -= children.length;
|
|
527
|
+
extractBudget -= extractedPages.length;
|
|
528
|
+
for (const created of [...children, ...extractedPages])
|
|
529
|
+
knownSlugs.add(created.slug.toLowerCase());
|
|
530
|
+
result.pages.push({
|
|
531
|
+
slug: row.slug,
|
|
532
|
+
mode: row.dream_management,
|
|
533
|
+
action: options.dryRun ? 'would-update' : 'updated',
|
|
534
|
+
splits: children.map((child) => child.slug),
|
|
535
|
+
extractions: extractedPages.map((page) => page.slug),
|
|
536
|
+
merges: [],
|
|
537
|
+
issues: [],
|
|
538
|
+
...temporalResult(temporal, temporalSource, clock, archival),
|
|
539
|
+
});
|
|
540
|
+
}
|
|
541
|
+
result.drafts = [
|
|
542
|
+
...mergeDrafts,
|
|
543
|
+
...staged.map((stage) => ({
|
|
544
|
+
slug: stage.row.slug,
|
|
545
|
+
mode: stage.row.dream_management,
|
|
546
|
+
relPath: stage.row.rel_path,
|
|
547
|
+
inputHash: stage.inputHash,
|
|
548
|
+
before: stage.before,
|
|
549
|
+
after: stage.after,
|
|
550
|
+
children: stage.children,
|
|
551
|
+
extractions: stage.extractions,
|
|
552
|
+
merge: null,
|
|
553
|
+
evidence: stage.evidence.map((entry) => ({
|
|
554
|
+
slug: entry.slug,
|
|
555
|
+
relPath: entry.rel_path,
|
|
556
|
+
relationship: entry.relationship,
|
|
557
|
+
bodyHash: entry.body_hash,
|
|
558
|
+
contentHash: entry.content_hash,
|
|
559
|
+
summary: entry.relationship === 'about' ? entry.summary : null,
|
|
560
|
+
claims: entry.facts.map((fact) => fact.claim),
|
|
561
|
+
events: entry.events.map((event) => `${event.date}: ${event.summary}`),
|
|
562
|
+
})),
|
|
563
|
+
conflicts: stage.conflicts.map((entry) => ({
|
|
564
|
+
slug: entry.slug,
|
|
565
|
+
subject: entry.subject,
|
|
566
|
+
attribute: entry.attribute,
|
|
567
|
+
claim: entry.claim,
|
|
568
|
+
value: entry.value,
|
|
569
|
+
})),
|
|
570
|
+
})),
|
|
571
|
+
];
|
|
572
|
+
if (options.dryRun) {
|
|
573
|
+
for (const draft of mergeDrafts) {
|
|
574
|
+
const row = rows.find((candidate) => candidate.slug === draft.slug);
|
|
575
|
+
if (row)
|
|
576
|
+
queueCurateState(state, row.id, draft.inputHash, 'preview');
|
|
577
|
+
}
|
|
578
|
+
for (const stage of staged) {
|
|
579
|
+
queueCurateState(state, stage.row.id, stage.inputHash, 'preview');
|
|
580
|
+
}
|
|
581
|
+
if (options.recordState)
|
|
582
|
+
persistCurateState(ctx, state.values());
|
|
583
|
+
return result;
|
|
584
|
+
}
|
|
585
|
+
if (staged.length === 0) {
|
|
586
|
+
if (options.recordState)
|
|
587
|
+
persistCurateState(ctx, state.values());
|
|
588
|
+
return result;
|
|
589
|
+
}
|
|
590
|
+
for (const stage of staged) {
|
|
591
|
+
const main = await writeFileAtomic(ctx.config.aknoPath, stage.row.rel_path, stage.after);
|
|
592
|
+
result.files.push(fileEntry(main));
|
|
593
|
+
for (const child of stage.children) {
|
|
594
|
+
const written = await writeFileAtomic(ctx.config.aknoPath, child.relPath, child.content);
|
|
595
|
+
result.files.push(fileEntry(written));
|
|
596
|
+
}
|
|
597
|
+
for (const extracted of stage.extractions) {
|
|
598
|
+
const written = await writeFileAtomic(ctx.config.aknoPath, extracted.relPath, extracted.content);
|
|
599
|
+
result.files.push(fileEntry(written));
|
|
600
|
+
}
|
|
601
|
+
}
|
|
602
|
+
result.changeId = ctx.journal.record({
|
|
603
|
+
actor: 'agent',
|
|
604
|
+
op: 'curate',
|
|
605
|
+
summary: `curate: ${staged.length} canonical page(s), ${splitLimit - splitBudget} split(s), ` +
|
|
606
|
+
`${extractLimit - extractBudget} extraction(s)`,
|
|
607
|
+
files: result.files,
|
|
608
|
+
});
|
|
609
|
+
const paths = result.files.map((file) => file.relPath);
|
|
610
|
+
await ctx.indexer.run({ only: paths, modelPaths: [] });
|
|
611
|
+
ctx.derive.schedule(paths);
|
|
612
|
+
const postExtractionPolicyHash = extractionPolicyFingerprint(ctx);
|
|
613
|
+
// The rewrite changes the canonical page's own hash. Record the post-write fingerprint or the
|
|
614
|
+
// curator would interpret its own work as new input on the next cycle. New split children are
|
|
615
|
+
// marked too, so creating one does not immediately enqueue it for another synthesis.
|
|
616
|
+
for (const stage of staged) {
|
|
617
|
+
if (stage.metadataOnly)
|
|
618
|
+
continue;
|
|
619
|
+
for (const slug of [
|
|
620
|
+
stage.row.slug,
|
|
621
|
+
...stage.children.map((child) => child.slug),
|
|
622
|
+
...stage.extractions.map((page) => page.slug),
|
|
623
|
+
]) {
|
|
624
|
+
const refreshed = pageForSlug(ctx, slug);
|
|
625
|
+
if (!refreshed)
|
|
626
|
+
continue;
|
|
627
|
+
const temporal = temporalForRow(refreshed);
|
|
628
|
+
const eventState = temporal ? temporalState(temporal, clock) : null;
|
|
629
|
+
const archival = refreshed.dream_management === 'synthesize' && eventState === 'past';
|
|
630
|
+
const allEvidence = refreshed.dream_management === 'synthesize' ? evidenceFor(ctx, refreshed) : [];
|
|
631
|
+
const evidence = archival ? archivalEvidence(allEvidence) : allEvidence;
|
|
632
|
+
const conflicts = refreshed.dream_management === 'synthesize' ? conflictsFor(ctx, refreshed.id) : [];
|
|
633
|
+
queueCurateState(state, refreshed.id, curateInputHash(refreshed, evidence, conflicts, temporal, eventState, postExtractionPolicyHash, incomingLinkFingerprint(ctx, refreshed.id)), 'applied');
|
|
634
|
+
}
|
|
635
|
+
}
|
|
636
|
+
if (options.recordState)
|
|
637
|
+
persistCurateState(ctx, state.values());
|
|
638
|
+
return result;
|
|
639
|
+
}
|
|
640
|
+
async function discoverMergeCandidates(ctx, rows, folders) {
|
|
641
|
+
const eligible = rows.filter((row) => pageAllowsMaintenanceTransform(ctx.config, { slug: row.slug, role: row.role, dreamManagement: row.dream_management }, 'merge') && mergePathAllowed(row.slug, folders));
|
|
642
|
+
if (eligible.length < 2 || folders.length === 0) {
|
|
643
|
+
return { candidates: [], degraded: [], warnings: [], semanticMerge: null };
|
|
644
|
+
}
|
|
645
|
+
const identities = new Map();
|
|
646
|
+
for (const row of eligible) {
|
|
647
|
+
for (const value of [row.slug, row.title]) {
|
|
648
|
+
const key = exactIdentityKey(value);
|
|
649
|
+
const found = identities.get(key) ?? [];
|
|
650
|
+
found.push(row);
|
|
651
|
+
identities.set(key, found);
|
|
652
|
+
}
|
|
653
|
+
}
|
|
654
|
+
const pairs = new Map();
|
|
655
|
+
for (const canonical of eligible) {
|
|
656
|
+
for (const alias of storedStrings(canonical.aliases)) {
|
|
657
|
+
const matches = [
|
|
658
|
+
...new Map((identities.get(exactIdentityKey(alias)) ?? [])
|
|
659
|
+
.filter((row) => row.id !== canonical.id)
|
|
660
|
+
.map((row) => [row.id, row])).values(),
|
|
661
|
+
];
|
|
662
|
+
if (matches.length !== 1)
|
|
663
|
+
continue;
|
|
664
|
+
const duplicate = matches[0];
|
|
665
|
+
const pairKey = semanticMergePairKey(canonical.id, duplicate.id);
|
|
666
|
+
const proposed = {
|
|
667
|
+
canonical,
|
|
668
|
+
duplicate,
|
|
669
|
+
identityKind: 'exact_alias',
|
|
670
|
+
identitySignal: `exact alias ${JSON.stringify(alias)} on ${canonical.slug} identifies ${duplicate.slug}`,
|
|
671
|
+
identityEvidence: [],
|
|
672
|
+
};
|
|
673
|
+
const prior = pairs.get(pairKey);
|
|
674
|
+
if (!prior) {
|
|
675
|
+
pairs.set(pairKey, proposed);
|
|
676
|
+
continue;
|
|
677
|
+
}
|
|
678
|
+
// Reciprocal aliases are equally explicit. Prefer the page with more authored bytes,
|
|
679
|
+
// then a stable slug tie-break, so two consecutive cycles cannot choose opposite sides.
|
|
680
|
+
const preferred = [prior.canonical, proposed.canonical].sort((left, right) => right.bytes - left.bytes || left.slug.localeCompare(right.slug))[0];
|
|
681
|
+
if (preferred.id === proposed.canonical.id) {
|
|
682
|
+
pairs.set(pairKey, { ...proposed, identitySignal: `reciprocal exact aliases identify one subject` });
|
|
683
|
+
}
|
|
684
|
+
else {
|
|
685
|
+
pairs.set(pairKey, { ...prior, identitySignal: `reciprocal exact aliases identify one subject` });
|
|
686
|
+
}
|
|
687
|
+
}
|
|
688
|
+
}
|
|
689
|
+
for (const proposed of graphSubjectMergeCandidates(ctx, eligible)) {
|
|
690
|
+
const pairKey = semanticMergePairKey(proposed.canonical.id, proposed.duplicate.id);
|
|
691
|
+
// An authored alias is stronger and already chooses the canonical destination.
|
|
692
|
+
if (!pairs.has(pairKey))
|
|
693
|
+
pairs.set(pairKey, proposed);
|
|
694
|
+
}
|
|
695
|
+
const selected = [];
|
|
696
|
+
const occupied = new Set();
|
|
697
|
+
for (const candidate of [...pairs.values()].sort((left, right) => Number(left.identityKind !== 'exact_alias') - Number(right.identityKind !== 'exact_alias') ||
|
|
698
|
+
right.identityEvidence.length - left.identityEvidence.length ||
|
|
699
|
+
left.canonical.slug.localeCompare(right.canonical.slug))) {
|
|
700
|
+
if (occupied.has(candidate.canonical.id) || occupied.has(candidate.duplicate.id))
|
|
701
|
+
continue;
|
|
702
|
+
occupied.add(candidate.canonical.id);
|
|
703
|
+
occupied.add(candidate.duplicate.id);
|
|
704
|
+
selected.push(candidate);
|
|
705
|
+
}
|
|
706
|
+
if (ctx.config.maintenance.curate.mergeDiscovery !== 'semantic') {
|
|
707
|
+
return { candidates: selected, degraded: [], warnings: [], semanticMerge: null };
|
|
708
|
+
}
|
|
709
|
+
const semantic = await discoverSemanticMergeCandidates(ctx, eligible
|
|
710
|
+
.filter((row) => !occupied.has(row.id))
|
|
711
|
+
.map((row) => ({
|
|
712
|
+
id: row.id,
|
|
713
|
+
slug: row.slug,
|
|
714
|
+
relPath: row.rel_path,
|
|
715
|
+
title: row.title,
|
|
716
|
+
bodyHash: row.body_hash,
|
|
717
|
+
bytes: row.bytes,
|
|
718
|
+
})), {
|
|
719
|
+
excludedPairKeys: new Set(pairs.keys()),
|
|
720
|
+
candidateLimit: Math.min(20, Math.max(ctx.config.maintenance.curate.maxMerges * 4, 1)),
|
|
721
|
+
});
|
|
722
|
+
for (const pair of semantic.pairs.sort((left, right) => right.score - left.score || left.canonical.slug.localeCompare(right.canonical.slug))) {
|
|
723
|
+
if (occupied.has(pair.canonical.id) || occupied.has(pair.duplicate.id))
|
|
724
|
+
continue;
|
|
725
|
+
occupied.add(pair.canonical.id);
|
|
726
|
+
occupied.add(pair.duplicate.id);
|
|
727
|
+
selected.push({
|
|
728
|
+
canonical: eligible.find((row) => row.id === pair.canonical.id),
|
|
729
|
+
duplicate: eligible.find((row) => row.id === pair.duplicate.id),
|
|
730
|
+
identityKind: 'semantic',
|
|
731
|
+
identitySignal: `cosine ${pair.score.toFixed(4)} passed the qualified semantic prefilter and ` +
|
|
732
|
+
`${SEMANTIC_MERGE_PROMPT_VERSION} classified the complete pages as one durable subject`,
|
|
733
|
+
identityEvidence: [],
|
|
734
|
+
});
|
|
735
|
+
}
|
|
736
|
+
return {
|
|
737
|
+
candidates: selected,
|
|
738
|
+
degraded: semantic.degraded,
|
|
739
|
+
warnings: semantic.warnings,
|
|
740
|
+
semanticMerge: semantic.metrics,
|
|
741
|
+
};
|
|
742
|
+
}
|
|
743
|
+
/**
|
|
744
|
+
* Broader merge discovery without a similarity threshold: a candidate page must carry at least two distinct,
|
|
745
|
+
* current fact attributes whose subjects resolve exactly to another eligible page's canonical entity. The
|
|
746
|
+
* candidate title must also contain that entity's complete title and live beside it. This catches durable
|
|
747
|
+
* subject pages such as "Ada Marlow field notes" without treating a shared template or embedding score as
|
|
748
|
+
* identity. The verifier and independent curator still decide whether the separate purpose is worth keeping.
|
|
749
|
+
*/
|
|
750
|
+
function graphSubjectMergeCandidates(ctx, eligible) {
|
|
751
|
+
if (eligible.length < 2)
|
|
752
|
+
return [];
|
|
753
|
+
const eligibleById = new Map(eligible.map((row) => [row.id, row]));
|
|
754
|
+
const rows = ctx.store.db
|
|
755
|
+
.prepare(`SELECT f.page_id, f.id AS fact_id, s.subject_entity, s.predicate, s.source_hash,
|
|
756
|
+
e.canonical_page, e.label
|
|
757
|
+
FROM graph_fact_status s
|
|
758
|
+
JOIN facts f ON f.id = s.fact_id
|
|
759
|
+
JOIN graph_entities e ON e.id = s.subject_entity
|
|
760
|
+
WHERE f.valid_to IS NULL
|
|
761
|
+
AND s.subject_resolution = 'exact'
|
|
762
|
+
AND s.eligibility = 'eligible'
|
|
763
|
+
AND s.traversable = 1
|
|
764
|
+
ORDER BY f.page_id, e.canonical_page, s.predicate, f.id`)
|
|
765
|
+
.all();
|
|
766
|
+
const groups = new Map();
|
|
767
|
+
for (const row of rows) {
|
|
768
|
+
if (row.page_id === row.canonical_page)
|
|
769
|
+
continue;
|
|
770
|
+
if (!eligibleById.has(row.page_id) || !eligibleById.has(row.canonical_page))
|
|
771
|
+
continue;
|
|
772
|
+
const key = `${row.canonical_page}\0${row.page_id}`;
|
|
773
|
+
const found = groups.get(key) ?? [];
|
|
774
|
+
found.push(row);
|
|
775
|
+
groups.set(key, found);
|
|
776
|
+
}
|
|
777
|
+
const candidates = [];
|
|
778
|
+
for (const evidence of groups.values()) {
|
|
779
|
+
const first = evidence[0];
|
|
780
|
+
const canonical = eligibleById.get(first.canonical_page);
|
|
781
|
+
const duplicate = eligibleById.get(first.page_id);
|
|
782
|
+
if (path.posix.dirname(canonical.slug) !== path.posix.dirname(duplicate.slug))
|
|
783
|
+
continue;
|
|
784
|
+
if (!titleContainsCompleteIdentity(duplicate.title, first.label))
|
|
785
|
+
continue;
|
|
786
|
+
const predicates = new Set(evidence.map((entry) => entry.predicate));
|
|
787
|
+
if (predicates.size < 2)
|
|
788
|
+
continue;
|
|
789
|
+
const identityEvidence = evidence
|
|
790
|
+
.filter((entry, index, all) => all.findIndex((candidate) => candidate.predicate === entry.predicate) === index)
|
|
791
|
+
.slice(0, 4)
|
|
792
|
+
.map((entry) => ({
|
|
793
|
+
factId: entry.fact_id,
|
|
794
|
+
predicate: entry.predicate,
|
|
795
|
+
sourceHash: entry.source_hash,
|
|
796
|
+
}));
|
|
797
|
+
candidates.push({
|
|
798
|
+
canonical,
|
|
799
|
+
duplicate,
|
|
800
|
+
identityKind: 'graph_subject',
|
|
801
|
+
identitySignal: `${JSON.stringify(first.label)} is the exact canonical entity for ${canonical.slug}; ` +
|
|
802
|
+
`${duplicate.slug} names that complete identity and has ${predicates.size} distinct current ` +
|
|
803
|
+
`attributes resolved exactly to it (${[...predicates]
|
|
804
|
+
.slice(0, 4)
|
|
805
|
+
.map((predicate) => JSON.stringify(predicate))
|
|
806
|
+
.join(', ')})`,
|
|
807
|
+
identityEvidence,
|
|
808
|
+
});
|
|
809
|
+
}
|
|
810
|
+
return candidates.sort((left, right) => right.identityEvidence.length - left.identityEvidence.length ||
|
|
811
|
+
left.canonical.slug.localeCompare(right.canonical.slug) ||
|
|
812
|
+
left.duplicate.slug.localeCompare(right.duplicate.slug));
|
|
813
|
+
}
|
|
814
|
+
function titleContainsCompleteIdentity(title, identity) {
|
|
815
|
+
const titleTokens = new Set(identityTokens(title));
|
|
816
|
+
const identityWords = identityTokens(identity);
|
|
817
|
+
return identityWords.length >= 2 && identityWords.every((token) => titleTokens.has(token));
|
|
818
|
+
}
|
|
819
|
+
function identityTokens(value) {
|
|
820
|
+
return (value
|
|
821
|
+
.normalize('NFKC')
|
|
822
|
+
.toLocaleLowerCase('und')
|
|
823
|
+
.match(/[\p{L}\p{N}]+/gu) ?? []);
|
|
824
|
+
}
|
|
825
|
+
function exactIdentityKey(value) {
|
|
826
|
+
return value
|
|
827
|
+
.trim()
|
|
828
|
+
.replace(/\\/g, '/')
|
|
829
|
+
.replace(/\.(?:md|markdown)$/i, '')
|
|
830
|
+
.normalize('NFKC')
|
|
831
|
+
.toLowerCase();
|
|
832
|
+
}
|
|
833
|
+
function storedStrings(value) {
|
|
834
|
+
try {
|
|
835
|
+
const parsed = JSON.parse(value);
|
|
836
|
+
return Array.isArray(parsed)
|
|
837
|
+
? parsed.filter((entry) => typeof entry === 'string' && entry.trim().length > 0)
|
|
838
|
+
: [];
|
|
839
|
+
}
|
|
840
|
+
catch {
|
|
841
|
+
return [];
|
|
842
|
+
}
|
|
843
|
+
}
|
|
844
|
+
async function inspectMergeCandidate(ctx, candidate) {
|
|
845
|
+
const [canonicalBefore, duplicateBefore] = await Promise.all([
|
|
846
|
+
fsp.readFile(path.join(ctx.config.aknoPath, candidate.canonical.rel_path), 'utf8').catch(() => null),
|
|
847
|
+
fsp.readFile(path.join(ctx.config.aknoPath, candidate.duplicate.rel_path), 'utf8').catch(() => null),
|
|
848
|
+
]);
|
|
849
|
+
if (canonicalBefore === null || duplicateBefore === null)
|
|
850
|
+
return null;
|
|
851
|
+
const canonical = parsePage(candidate.canonical.rel_path, canonicalBefore);
|
|
852
|
+
const duplicate = parsePage(candidate.duplicate.rel_path, duplicateBefore);
|
|
853
|
+
const conflicts = [
|
|
854
|
+
...conflictsFor(ctx, candidate.canonical.id),
|
|
855
|
+
...conflictsFor(ctx, candidate.duplicate.id),
|
|
856
|
+
];
|
|
857
|
+
const inbound = await mergeInboundPages(ctx, candidate.duplicate);
|
|
858
|
+
const inputHash = mergeInputHash(ctx, candidate, canonicalBefore, duplicateBefore, inbound, conflicts);
|
|
859
|
+
const issues = mergeEligibilityIssues(ctx, candidate, canonical, duplicate, inbound, conflicts);
|
|
860
|
+
const inverseIssue = inverseSplitMergeIssue(ctx, { relPath: candidate.canonical.rel_path, content: canonicalBefore }, { relPath: candidate.duplicate.rel_path, content: duplicateBefore });
|
|
861
|
+
if (inverseIssue)
|
|
862
|
+
issues.push(inverseIssue);
|
|
863
|
+
if (Buffer.byteLength(canonical.body) + Buffer.byteLength(duplicate.body) > 80_000) {
|
|
864
|
+
issues.push('merge inputs exceed the 80000-byte lossless planning limit');
|
|
865
|
+
}
|
|
866
|
+
const duplicateAliases = storedStrings(candidate.duplicate.aliases);
|
|
867
|
+
const aliasCollision = mergeAliasCollision(ctx, candidate, [
|
|
868
|
+
duplicate.slug,
|
|
869
|
+
duplicate.title,
|
|
870
|
+
...duplicateAliases,
|
|
871
|
+
]);
|
|
872
|
+
if (aliasCollision)
|
|
873
|
+
issues.push(aliasCollision);
|
|
874
|
+
const canonicalWithAliases = withAknoAliases(canonicalBefore, [
|
|
875
|
+
duplicate.slug,
|
|
876
|
+
duplicate.title,
|
|
877
|
+
...duplicateAliases,
|
|
878
|
+
]);
|
|
879
|
+
if (canonicalWithAliases === null) {
|
|
880
|
+
issues.push('canonical frontmatter cannot accept aliases without reformatting unknown YAML');
|
|
881
|
+
}
|
|
882
|
+
return {
|
|
883
|
+
candidate,
|
|
884
|
+
canonicalBefore,
|
|
885
|
+
duplicateBefore,
|
|
886
|
+
canonical,
|
|
887
|
+
duplicate,
|
|
888
|
+
inbound,
|
|
889
|
+
inputHash,
|
|
890
|
+
canonicalWithAliases,
|
|
891
|
+
issues: [...new Set(issues)],
|
|
892
|
+
};
|
|
893
|
+
}
|
|
894
|
+
async function prepareMergeDraft(ctx, inspection) {
|
|
895
|
+
const { candidate, canonicalBefore, duplicateBefore, canonical, duplicate, inbound, inputHash, canonicalWithAliases, issues, } = inspection;
|
|
896
|
+
if (issues.length > 0 || canonicalWithAliases === null) {
|
|
897
|
+
return { inputHash, draft: null, issues, cacheable: true };
|
|
898
|
+
}
|
|
899
|
+
const canonicalPrepared = rewritePageLinks(canonical.body, canonical.slug, duplicate.slug, canonical.slug);
|
|
900
|
+
const duplicatePrepared = rewritePageLinks(withoutDuplicateTitle(duplicate.body, duplicate.title), duplicate.slug, duplicate.slug, canonical.slug);
|
|
901
|
+
const planned = await ctx.models.derive.chat([
|
|
902
|
+
{ role: 'system', content: MERGE_SYSTEM },
|
|
903
|
+
{
|
|
904
|
+
role: 'user',
|
|
905
|
+
content: `Identity signal: ${candidate.identitySignal}\nCanonical slug: ${canonical.slug}\n` +
|
|
906
|
+
`Duplicate slug to retire: ${duplicate.slug}\n\nCanonical body:\n${canonicalPrepared}\n\n` +
|
|
907
|
+
`Prepared duplicate body:\n${duplicatePrepared}`,
|
|
908
|
+
},
|
|
909
|
+
], { schema: MERGE_SCHEMA, maxTokens: 8_000 });
|
|
910
|
+
const parsed = planned.ok && planned.value ? parseJsonLoose(planned.value) : null;
|
|
911
|
+
const nextBody = typeof parsed?.body === 'string' ? endWithNewline(parsed.body) : null;
|
|
912
|
+
if (!nextBody) {
|
|
913
|
+
if (planned.ok)
|
|
914
|
+
ctx.models.derive.reportInvalidResponse();
|
|
915
|
+
return {
|
|
916
|
+
inputHash,
|
|
917
|
+
draft: null,
|
|
918
|
+
issues: [planned.error ?? 'merge planner returned invalid JSON without a body'],
|
|
919
|
+
cacheable: planned.ok,
|
|
920
|
+
};
|
|
921
|
+
}
|
|
922
|
+
const guarded = mergeAccountingIssues(canonicalPrepared, duplicatePrepared, nextBody);
|
|
923
|
+
const incomingAnchors = await incomingHeadingAnchors(ctx, candidate.duplicate.id, duplicate.slug);
|
|
924
|
+
const resultingHeadings = headingReferences(nextBody);
|
|
925
|
+
if ([...incomingAnchors].some((anchor) => !resultingHeadings.has(anchor))) {
|
|
926
|
+
guarded.push('the merge would remove a heading targeted by an incoming link');
|
|
927
|
+
}
|
|
928
|
+
if (guarded.length > 0) {
|
|
929
|
+
return { inputHash, draft: null, issues: [...new Set(guarded)], cacheable: true };
|
|
930
|
+
}
|
|
931
|
+
const aliasFm = parseFrontmatter(canonicalWithAliases);
|
|
932
|
+
const canonicalAfter = canonicalWithAliases.slice(0, aliasFm.bodyOffset) + nextBody;
|
|
933
|
+
const linkUpdates = inbound
|
|
934
|
+
.filter((page) => page.id !== candidate.canonical.id)
|
|
935
|
+
.map((page) => ({
|
|
936
|
+
slug: page.slug,
|
|
937
|
+
relPath: page.relPath,
|
|
938
|
+
before: page.content,
|
|
939
|
+
after: rewritePageLinks(page.content, page.slug, duplicate.slug, canonical.slug),
|
|
940
|
+
}));
|
|
941
|
+
if (linkUpdates.some((update) => update.before === update.after)) {
|
|
942
|
+
return {
|
|
943
|
+
inputHash,
|
|
944
|
+
draft: null,
|
|
945
|
+
issues: ['an indexed inbound link could not be rewritten exactly'],
|
|
946
|
+
cacheable: true,
|
|
947
|
+
};
|
|
948
|
+
}
|
|
949
|
+
const verified = await verifyMergeDraft(ctx, candidate, canonicalPrepared, duplicatePrepared, canonicalAfter, linkUpdates);
|
|
950
|
+
if (!verified.ok) {
|
|
951
|
+
return { inputHash, draft: null, issues: verified.issues, cacheable: verified.cacheable };
|
|
952
|
+
}
|
|
953
|
+
return {
|
|
954
|
+
inputHash,
|
|
955
|
+
issues: [],
|
|
956
|
+
cacheable: true,
|
|
957
|
+
draft: {
|
|
958
|
+
slug: canonical.slug,
|
|
959
|
+
mode: 'synthesize',
|
|
960
|
+
relPath: canonical.relPath,
|
|
961
|
+
inputHash,
|
|
962
|
+
before: canonicalBefore,
|
|
963
|
+
after: canonicalAfter,
|
|
964
|
+
children: [],
|
|
965
|
+
extractions: [],
|
|
966
|
+
merge: {
|
|
967
|
+
sourceSlug: duplicate.slug,
|
|
968
|
+
sourceRelPath: duplicate.relPath,
|
|
969
|
+
sourceBefore: duplicateBefore,
|
|
970
|
+
sourceBodyHash: candidate.duplicate.body_hash,
|
|
971
|
+
identitySignal: candidate.identitySignal,
|
|
972
|
+
identityKind: candidate.identityKind,
|
|
973
|
+
linkUpdates,
|
|
974
|
+
},
|
|
975
|
+
evidence: [],
|
|
976
|
+
conflicts: [],
|
|
977
|
+
},
|
|
978
|
+
};
|
|
979
|
+
}
|
|
980
|
+
async function mergeInboundPages(ctx, duplicate) {
|
|
981
|
+
const rows = ctx.store.db
|
|
982
|
+
.prepare(`SELECT DISTINCT p.id, p.slug, p.rel_path, p.role, p.dream_management, p.body_hash
|
|
983
|
+
FROM links l JOIN pages p ON p.id = l.from_page
|
|
984
|
+
WHERE lower(l.to_slug) = lower(?) AND l.from_page != ? AND l.kind != 'embed'
|
|
985
|
+
ORDER BY p.slug`)
|
|
986
|
+
.all(duplicate.slug, duplicate.id);
|
|
987
|
+
const pages = [];
|
|
988
|
+
for (const row of rows) {
|
|
989
|
+
const content = await fsp
|
|
990
|
+
.readFile(path.join(ctx.config.aknoPath, row.rel_path), 'utf8')
|
|
991
|
+
.catch(() => null);
|
|
992
|
+
if (content === null)
|
|
993
|
+
continue;
|
|
994
|
+
pages.push({
|
|
995
|
+
id: row.id,
|
|
996
|
+
slug: row.slug,
|
|
997
|
+
relPath: row.rel_path,
|
|
998
|
+
role: row.role,
|
|
999
|
+
dreamManagement: row.dream_management,
|
|
1000
|
+
bodyHash: row.body_hash,
|
|
1001
|
+
content,
|
|
1002
|
+
});
|
|
1003
|
+
}
|
|
1004
|
+
return pages;
|
|
1005
|
+
}
|
|
1006
|
+
function mergeEligibilityIssues(ctx, candidate, canonical, duplicate, inbound, conflicts) {
|
|
1007
|
+
const issues = [];
|
|
1008
|
+
if (canonical.declaredManagement.dream !== 'synthesize' ||
|
|
1009
|
+
duplicate.declaredManagement.dream !== 'synthesize') {
|
|
1010
|
+
issues.push('both merge pages must explicitly declare dream: synthesize');
|
|
1011
|
+
}
|
|
1012
|
+
if (canonical.declaredRole && canonical.declaredRole !== 'knowledge') {
|
|
1013
|
+
issues.push('the canonical merge page is not declared as knowledge');
|
|
1014
|
+
}
|
|
1015
|
+
if (duplicate.declaredRole && duplicate.declaredRole !== 'knowledge') {
|
|
1016
|
+
issues.push('the duplicate merge page is not declared as knowledge');
|
|
1017
|
+
}
|
|
1018
|
+
if (canonical.about.includes(duplicate.slug) || duplicate.about.includes(canonical.slug)) {
|
|
1019
|
+
issues.push('parent/child pages cannot be merged as duplicate identities');
|
|
1020
|
+
}
|
|
1021
|
+
if (conflicts.length > 0)
|
|
1022
|
+
issues.push('unresolved conflicts must be handled before these pages can merge');
|
|
1023
|
+
const documents = ctx.store.db
|
|
1024
|
+
.prepare('SELECT count(*) AS n FROM documents WHERE page_id = ?')
|
|
1025
|
+
.get(candidate.duplicate.id);
|
|
1026
|
+
if (documents.n > 0)
|
|
1027
|
+
issues.push('the duplicate owns documents whose canonical ownership is unresolved');
|
|
1028
|
+
for (const page of inbound) {
|
|
1029
|
+
if (page.id === candidate.canonical.id)
|
|
1030
|
+
continue;
|
|
1031
|
+
if (page.role !== 'knowledge' || page.dreamManagement !== 'synthesize') {
|
|
1032
|
+
issues.push(`inbound link page ${page.slug} is not opted in to synthesis link updates`);
|
|
1033
|
+
continue;
|
|
1034
|
+
}
|
|
1035
|
+
const parsed = parsePage(page.relPath, page.content);
|
|
1036
|
+
if (parsed.declaredManagement.dream !== 'synthesize') {
|
|
1037
|
+
issues.push(`inbound link page ${page.slug} does not explicitly permit synthesis writes`);
|
|
1038
|
+
}
|
|
1039
|
+
}
|
|
1040
|
+
issues.push(...mergeFrontmatterIssues(canonical.frontmatter.data, duplicate.frontmatter.data));
|
|
1041
|
+
return issues;
|
|
1042
|
+
}
|
|
1043
|
+
function mergeFrontmatterIssues(canonical, duplicate) {
|
|
1044
|
+
const issues = [];
|
|
1045
|
+
for (const [key, value] of Object.entries(duplicate)) {
|
|
1046
|
+
if (key === 'title' || key === 'id' || key === 'akno')
|
|
1047
|
+
continue;
|
|
1048
|
+
if (JSON.stringify(canonical[key]) !== JSON.stringify(value)) {
|
|
1049
|
+
issues.push(`duplicate frontmatter key ${key} has no lossless canonical disposition`);
|
|
1050
|
+
}
|
|
1051
|
+
}
|
|
1052
|
+
const canonicalAkno = objectValue(canonical.akno);
|
|
1053
|
+
const duplicateAkno = objectValue(duplicate.akno);
|
|
1054
|
+
for (const [key, value] of Object.entries(duplicateAkno)) {
|
|
1055
|
+
if (key === 'aliases' || key === 'role')
|
|
1056
|
+
continue;
|
|
1057
|
+
if (key === 'about') {
|
|
1058
|
+
const canonicalAbout = new Set(stringArray(canonicalAkno.about));
|
|
1059
|
+
if (stringArray(value).some((entry) => !canonicalAbout.has(entry))) {
|
|
1060
|
+
issues.push('duplicate about relationships have no lossless canonical disposition');
|
|
1061
|
+
}
|
|
1062
|
+
continue;
|
|
1063
|
+
}
|
|
1064
|
+
if (JSON.stringify(canonicalAkno[key]) !== JSON.stringify(value)) {
|
|
1065
|
+
issues.push(`duplicate akno.${key} metadata differs from the canonical page`);
|
|
1066
|
+
}
|
|
1067
|
+
}
|
|
1068
|
+
return issues;
|
|
1069
|
+
}
|
|
1070
|
+
function objectValue(value) {
|
|
1071
|
+
return value && typeof value === 'object' && !Array.isArray(value)
|
|
1072
|
+
? value
|
|
1073
|
+
: {};
|
|
1074
|
+
}
|
|
1075
|
+
function stringArray(value) {
|
|
1076
|
+
return Array.isArray(value) ? value.filter((entry) => typeof entry === 'string') : [];
|
|
1077
|
+
}
|
|
1078
|
+
function mergeAliasCollision(ctx, candidate, aliases) {
|
|
1079
|
+
const other = ctx.store.db
|
|
1080
|
+
.prepare('SELECT slug, title FROM pages WHERE id NOT IN (?, ?)')
|
|
1081
|
+
.all(candidate.canonical.id, candidate.duplicate.id);
|
|
1082
|
+
const occupied = new Set(other.flatMap((row) => [exactIdentityKey(row.slug), exactIdentityKey(row.title)]));
|
|
1083
|
+
const collision = aliases.find((alias) => occupied.has(exactIdentityKey(alias)));
|
|
1084
|
+
return collision ? `retired identity ${JSON.stringify(collision)} also identifies another page` : null;
|
|
1085
|
+
}
|
|
1086
|
+
function mergeInputHash(ctx, candidate, canonicalBefore, duplicateBefore, inbound, conflicts) {
|
|
1087
|
+
const documents = ctx.store.db
|
|
1088
|
+
.prepare('SELECT rel_path, sha256 FROM documents WHERE page_id = ? ORDER BY rel_path')
|
|
1089
|
+
.all(candidate.duplicate.id);
|
|
1090
|
+
const retiredKeys = new Set([candidate.duplicate.slug, candidate.duplicate.title, ...storedStrings(candidate.duplicate.aliases)].map(exactIdentityKey));
|
|
1091
|
+
const identityCollisions = ctx.store.db
|
|
1092
|
+
.prepare('SELECT id, slug, title FROM pages WHERE id NOT IN (?, ?) ORDER BY slug')
|
|
1093
|
+
.all(candidate.canonical.id, candidate.duplicate.id).filter((row) => retiredKeys.has(exactIdentityKey(row.slug)) || retiredKeys.has(exactIdentityKey(row.title)));
|
|
1094
|
+
return sha256(JSON.stringify({
|
|
1095
|
+
version: CURATE_FINGERPRINT_VERSION,
|
|
1096
|
+
kind: 'merge',
|
|
1097
|
+
inverseSplitGuard: 1,
|
|
1098
|
+
canonical: { slug: candidate.canonical.slug, hash: sha256(canonicalBefore) },
|
|
1099
|
+
duplicate: { slug: candidate.duplicate.slug, hash: sha256(duplicateBefore) },
|
|
1100
|
+
identitySignal: candidate.identitySignal,
|
|
1101
|
+
identityKind: candidate.identityKind,
|
|
1102
|
+
identityEvidence: candidate.identityEvidence,
|
|
1103
|
+
inbound: inbound.map((page) => ({
|
|
1104
|
+
slug: page.slug,
|
|
1105
|
+
hash: sha256(page.content),
|
|
1106
|
+
role: page.role,
|
|
1107
|
+
dreamManagement: page.dreamManagement,
|
|
1108
|
+
})),
|
|
1109
|
+
documents,
|
|
1110
|
+
identityCollisions,
|
|
1111
|
+
conflicts,
|
|
1112
|
+
policy: {
|
|
1113
|
+
maxMerges: ctx.config.maintenance.curate.maxMerges,
|
|
1114
|
+
mergeFolders: ctx.config.maintenance.curate.mergeFolders,
|
|
1115
|
+
mergeDiscovery: ctx.config.maintenance.curate.mergeDiscovery,
|
|
1116
|
+
},
|
|
1117
|
+
}));
|
|
1118
|
+
}
|
|
1119
|
+
/**
|
|
1120
|
+
* A merge must not immediately collapse sibling pages produced by one still-current split.
|
|
1121
|
+
* The change journal outlives private plan payloads, and exact after-bytes release this guard as
|
|
1122
|
+
* soon as either page acquires an independently edited purpose.
|
|
1123
|
+
*/
|
|
1124
|
+
export function inverseSplitMergeIssue(ctx, left, right) {
|
|
1125
|
+
const change = ctx.store.db
|
|
1126
|
+
.prepare(`SELECT change.id
|
|
1127
|
+
FROM changes change
|
|
1128
|
+
JOIN change_files left_file ON left_file.change_id = change.id
|
|
1129
|
+
JOIN change_files right_file ON right_file.change_id = change.id
|
|
1130
|
+
WHERE change.status = 'applied'
|
|
1131
|
+
AND change.op = 'maintenance'
|
|
1132
|
+
AND change.summary LIKE 'maintenance split:%'
|
|
1133
|
+
AND left_file.rel_path = ?
|
|
1134
|
+
AND left_file.action = 'created'
|
|
1135
|
+
AND left_file.after = ?
|
|
1136
|
+
AND right_file.rel_path = ?
|
|
1137
|
+
AND right_file.action = 'created'
|
|
1138
|
+
AND right_file.after = ?
|
|
1139
|
+
AND EXISTS (
|
|
1140
|
+
SELECT 1 FROM change_files canonical
|
|
1141
|
+
WHERE canonical.change_id = change.id AND canonical.action = 'modified'
|
|
1142
|
+
)
|
|
1143
|
+
ORDER BY change.rowid DESC
|
|
1144
|
+
LIMIT 1`)
|
|
1145
|
+
.get(left.relPath, left.content, right.relPath, right.content);
|
|
1146
|
+
return change
|
|
1147
|
+
? `merge would reverse applied split ${change.id} while both sibling outputs are unchanged; undo that split or let the pages evolve before reconsidering their identity`
|
|
1148
|
+
: null;
|
|
1149
|
+
}
|
|
1150
|
+
function withoutDuplicateTitle(body, title) {
|
|
1151
|
+
const lines = body.replaceAll('\r\n', '\n').split('\n');
|
|
1152
|
+
const index = lines.findIndex((line) => line.trim().length > 0);
|
|
1153
|
+
if (index < 0)
|
|
1154
|
+
return body;
|
|
1155
|
+
const heading = /^#\s+(.+?)\s*#*\s*$/.exec(lines[index]);
|
|
1156
|
+
if (!heading || exactIdentityKey(heading[1]) !== exactIdentityKey(title))
|
|
1157
|
+
return body;
|
|
1158
|
+
lines.splice(index, 1);
|
|
1159
|
+
while (lines[0] === '')
|
|
1160
|
+
lines.shift();
|
|
1161
|
+
return endWithNewline(lines.join('\n'));
|
|
1162
|
+
}
|
|
1163
|
+
function rewritePageLinks(text, fromPage, retired, canonical) {
|
|
1164
|
+
const wiki = text.replace(/\[\[([^\]|#]+)((?:#[^\]|]+)?(?:\|[^\]]+)?)\]\]/g, (whole, target, suffix) => {
|
|
1165
|
+
return normalizeLinkTarget(String(target), fromPage).toLowerCase() === retired.toLowerCase()
|
|
1166
|
+
? `[[${canonical}${String(suffix)}]]`
|
|
1167
|
+
: whole;
|
|
1168
|
+
});
|
|
1169
|
+
return wiki.replace(/(?<!!)\[([^\]]*)\]\(\s*<?([^\s)>]+)>?(\s+[^)]*)?\)/g, (whole, label, href, titlePart) => {
|
|
1170
|
+
const value = String(href);
|
|
1171
|
+
const hash = value.indexOf('#');
|
|
1172
|
+
const target = hash >= 0 ? value.slice(0, hash) : value;
|
|
1173
|
+
const fragment = hash >= 0 ? value.slice(hash) : '';
|
|
1174
|
+
if (normalizeLinkTarget(target, fromPage).toLowerCase() !== retired.toLowerCase())
|
|
1175
|
+
return whole;
|
|
1176
|
+
return `[${String(label)}](${canonical}.md${fragment}${String(titlePart ?? '')})`;
|
|
1177
|
+
});
|
|
1178
|
+
}
|
|
1179
|
+
function mergeAccountingIssues(canonical, duplicate, after) {
|
|
1180
|
+
const expected = nonBlankLineCounts(canonical);
|
|
1181
|
+
for (const [line, count] of nonBlankLineCounts(duplicate)) {
|
|
1182
|
+
expected.set(line, Math.max(expected.get(line) ?? 0, count));
|
|
1183
|
+
}
|
|
1184
|
+
const actual = nonBlankLineCounts(after);
|
|
1185
|
+
const issues = [];
|
|
1186
|
+
for (const [line, count] of expected) {
|
|
1187
|
+
if ((actual.get(line) ?? 0) !== count) {
|
|
1188
|
+
issues.push('merge did not preserve every unique authored line exactly once');
|
|
1189
|
+
break;
|
|
1190
|
+
}
|
|
1191
|
+
}
|
|
1192
|
+
if ([...actual].some(([line, count]) => !expected.has(line) || count !== expected.get(line))) {
|
|
1193
|
+
issues.push('merge body contains text that was not present in either source page');
|
|
1194
|
+
}
|
|
1195
|
+
const afterLines = nonBlankLines(after);
|
|
1196
|
+
if (!lineSubsequence(nonBlankLines(canonical), afterLines) ||
|
|
1197
|
+
!lineSubsequence(nonBlankLines(duplicate), afterLines)) {
|
|
1198
|
+
issues.push('merge changed the authored line order inside a source page');
|
|
1199
|
+
}
|
|
1200
|
+
for (const [marker, knowledge] of [...markerAttachments(canonical), ...markerAttachments(duplicate)]) {
|
|
1201
|
+
const markerIndex = afterLines.indexOf(marker);
|
|
1202
|
+
if (markerIndex < 0 || afterLines[markerIndex + 1] !== knowledge) {
|
|
1203
|
+
issues.push('merge detached a stable item marker from its authored knowledge');
|
|
1204
|
+
break;
|
|
1205
|
+
}
|
|
1206
|
+
}
|
|
1207
|
+
if (firstH1(after) !== firstH1(canonical)) {
|
|
1208
|
+
issues.push('merge changed the canonical page title heading');
|
|
1209
|
+
}
|
|
1210
|
+
return [...new Set(issues)];
|
|
1211
|
+
}
|
|
1212
|
+
function nonBlankLines(value) {
|
|
1213
|
+
return value
|
|
1214
|
+
.replaceAll('\r\n', '\n')
|
|
1215
|
+
.split('\n')
|
|
1216
|
+
.map((line) => line.trimEnd())
|
|
1217
|
+
.filter((line) => line.trim().length > 0);
|
|
1218
|
+
}
|
|
1219
|
+
function lineSubsequence(source, combined) {
|
|
1220
|
+
let at = 0;
|
|
1221
|
+
for (const line of source) {
|
|
1222
|
+
while (at < combined.length && combined[at] !== line)
|
|
1223
|
+
at++;
|
|
1224
|
+
if (at >= combined.length)
|
|
1225
|
+
return false;
|
|
1226
|
+
at++;
|
|
1227
|
+
}
|
|
1228
|
+
return true;
|
|
1229
|
+
}
|
|
1230
|
+
function markerAttachments(value) {
|
|
1231
|
+
const lines = nonBlankLines(value);
|
|
1232
|
+
const pairs = [];
|
|
1233
|
+
for (let index = 0; index < lines.length - 1; index++) {
|
|
1234
|
+
if (AKNO_ITEM.test(lines[index]))
|
|
1235
|
+
pairs.push([lines[index], lines[index + 1]]);
|
|
1236
|
+
}
|
|
1237
|
+
return pairs;
|
|
1238
|
+
}
|
|
1239
|
+
async function verifyMergeDraft(ctx, candidate, canonicalBody, duplicateBody, canonicalAfter, linkUpdates) {
|
|
1240
|
+
const result = await ctx.models.derive.chat([
|
|
1241
|
+
{ role: 'system', content: VERIFY_MERGE_SYSTEM },
|
|
1242
|
+
{
|
|
1243
|
+
role: 'user',
|
|
1244
|
+
content: JSON.stringify({
|
|
1245
|
+
identity: candidate.identitySignal,
|
|
1246
|
+
canonical: { slug: candidate.canonical.slug, before: canonicalBody, after: canonicalAfter },
|
|
1247
|
+
duplicate: { slug: candidate.duplicate.slug, body: duplicateBody, operation: 'delete' },
|
|
1248
|
+
inboundLinkUpdates: linkUpdates,
|
|
1249
|
+
}).slice(0, 100_000),
|
|
1250
|
+
},
|
|
1251
|
+
], { schema: VERIFY_SCHEMA, maxTokens: 1_200 });
|
|
1252
|
+
const parsed = result.ok && result.value ? parseJsonLoose(result.value) : null;
|
|
1253
|
+
const validIssues = Array.isArray(parsed?.issues) && parsed.issues.every((issue) => typeof issue === 'string');
|
|
1254
|
+
if (parsed?.ok === true && validIssues)
|
|
1255
|
+
return { ok: true, issues: [], cacheable: true };
|
|
1256
|
+
if (parsed?.ok === false && validIssues) {
|
|
1257
|
+
const issues = parsed.issues;
|
|
1258
|
+
return {
|
|
1259
|
+
ok: false,
|
|
1260
|
+
issues: issues.length > 0 ? issues : ['merge verifier rejected draft'],
|
|
1261
|
+
cacheable: true,
|
|
1262
|
+
};
|
|
1263
|
+
}
|
|
1264
|
+
if (result.ok)
|
|
1265
|
+
ctx.models.derive.reportInvalidResponse();
|
|
1266
|
+
return {
|
|
1267
|
+
ok: false,
|
|
1268
|
+
issues: [result.error ?? 'merge verifier returned invalid JSON'],
|
|
1269
|
+
cacheable: result.ok,
|
|
1270
|
+
};
|
|
1271
|
+
}
|
|
1272
|
+
function evidenceFor(ctx, page) {
|
|
1273
|
+
const rows = ctx.store.db
|
|
1274
|
+
.prepare(`SELECT DISTINCT p.id, p.slug, p.rel_path, p.summary, p.about, p.role, p.body_hash,
|
|
1275
|
+
indexed_file.sha256 AS content_hash,
|
|
1276
|
+
EXISTS (SELECT 1 FROM links l WHERE l.from_page = ? AND l.to_page = p.id) AS outbound,
|
|
1277
|
+
EXISTS (SELECT 1 FROM links l WHERE l.from_page = p.id AND l.to_page = ?) AS backlink
|
|
1278
|
+
FROM pages p JOIN files indexed_file ON indexed_file.rel_path = p.rel_path
|
|
1279
|
+
WHERE p.id != ? AND (
|
|
1280
|
+
EXISTS (SELECT 1 FROM links l WHERE l.from_page = p.id AND l.to_page = ?)
|
|
1281
|
+
OR EXISTS (SELECT 1 FROM links l WHERE l.from_page = ? AND l.to_page = p.id)
|
|
1282
|
+
OR p.about LIKE ?
|
|
1283
|
+
) ORDER BY p.slug COLLATE NOCASE LIMIT 30`)
|
|
1284
|
+
.all(page.id, page.id, page.id, page.id, page.id, `%${JSON.stringify(page.slug).slice(1, -1)}%`);
|
|
1285
|
+
const facts = ctx.store.db.prepare(`SELECT f.claim, f.subject, f.attribute, f.value, f.item_id FROM facts f
|
|
1286
|
+
JOIN pages current_page ON current_page.id = f.page_id
|
|
1287
|
+
WHERE f.page_id = ? AND f.valid_to IS NULL
|
|
1288
|
+
AND current_page.derived_hash = current_page.body_hash
|
|
1289
|
+
ORDER BY f.line_start, f.id LIMIT 50`);
|
|
1290
|
+
const events = ctx.store.db.prepare(`SELECT date, summary FROM events
|
|
1291
|
+
WHERE source_page = ? AND target_slug = ?
|
|
1292
|
+
ORDER BY date DESC, line LIMIT 50`);
|
|
1293
|
+
return rows.map((row) => {
|
|
1294
|
+
const relationship = pageRelationship(row, page.slug);
|
|
1295
|
+
const allFacts = facts.all(row.id);
|
|
1296
|
+
return {
|
|
1297
|
+
...row,
|
|
1298
|
+
relationship,
|
|
1299
|
+
events: events.all(row.id, page.slug),
|
|
1300
|
+
facts: relationship === 'about'
|
|
1301
|
+
? allFacts
|
|
1302
|
+
: relationship === 'backlink'
|
|
1303
|
+
? allFacts.filter((fact) => factMentionsPage(fact, page))
|
|
1304
|
+
: [],
|
|
1305
|
+
};
|
|
1306
|
+
});
|
|
1307
|
+
}
|
|
1308
|
+
/** Ended events wake only for evidence that explicitly contributes to or records the event. */
|
|
1309
|
+
function archivalEvidence(evidence) {
|
|
1310
|
+
return evidence.filter((row) => row.relationship === 'about' || row.facts.length > 0 || row.events.length > 0);
|
|
1311
|
+
}
|
|
1312
|
+
function pageRelationship(row, canonicalSlug) {
|
|
1313
|
+
try {
|
|
1314
|
+
const about = JSON.parse(row.about);
|
|
1315
|
+
if (Array.isArray(about) &&
|
|
1316
|
+
about.some((entry) => typeof entry === 'string' && normalizeLinkTarget(entry) === canonicalSlug)) {
|
|
1317
|
+
return 'about';
|
|
1318
|
+
}
|
|
1319
|
+
}
|
|
1320
|
+
catch {
|
|
1321
|
+
// Indexed policy JSON is generated by Akno; an unparseable row degrades to link relevance.
|
|
1322
|
+
}
|
|
1323
|
+
return row.outbound ? 'outbound' : 'backlink';
|
|
1324
|
+
}
|
|
1325
|
+
function factMentionsPage(fact, page) {
|
|
1326
|
+
const keys = [page.title, page.slug.split('/').at(-1)?.replaceAll('-', ' ') ?? '']
|
|
1327
|
+
.map(searchIdentity)
|
|
1328
|
+
.filter((value) => value.length >= 4);
|
|
1329
|
+
const text = searchIdentity([fact.subject, fact.attribute, fact.claim, fact.value].filter(Boolean).join(' '));
|
|
1330
|
+
return keys.some((key) => text.includes(key));
|
|
1331
|
+
}
|
|
1332
|
+
function searchIdentity(value) {
|
|
1333
|
+
return value
|
|
1334
|
+
.normalize('NFKC')
|
|
1335
|
+
.toLowerCase()
|
|
1336
|
+
.replace(/[^\p{L}\p{N}]+/gu, ' ')
|
|
1337
|
+
.trim();
|
|
1338
|
+
}
|
|
1339
|
+
function conflictsFor(ctx, pageId) {
|
|
1340
|
+
const rows = ctx.store.db
|
|
1341
|
+
.prepare(`SELECT f.subject, f.attribute, f.claim, f.value, p.slug FROM facts f
|
|
1342
|
+
JOIN pages p ON p.id = f.page_id
|
|
1343
|
+
WHERE f.valid_to IS NULL AND f.subject IS NOT NULL AND f.attribute IS NOT NULL
|
|
1344
|
+
AND p.derived_hash = p.body_hash
|
|
1345
|
+
AND EXISTS (
|
|
1346
|
+
SELECT 1 FROM facts other JOIN pages other_page ON other_page.id = other.page_id
|
|
1347
|
+
WHERE other.valid_to IS NULL AND other.page_id != f.page_id
|
|
1348
|
+
AND other_page.derived_hash = other_page.body_hash
|
|
1349
|
+
AND lower(other.subject) = lower(f.subject)
|
|
1350
|
+
AND lower(other.attribute) = lower(f.attribute)
|
|
1351
|
+
AND other.value != f.value
|
|
1352
|
+
)
|
|
1353
|
+
AND (f.page_id = ? OR p.about LIKE (SELECT '%' || slug || '%' FROM pages WHERE id = ?))
|
|
1354
|
+
ORDER BY p.slug COLLATE NOCASE, f.subject COLLATE NOCASE, f.attribute COLLATE NOCASE, f.claim
|
|
1355
|
+
LIMIT 20`)
|
|
1356
|
+
.all(pageId, pageId);
|
|
1357
|
+
return rows;
|
|
1358
|
+
}
|
|
1359
|
+
async function verifyDraft(ctx, page, before, after, splits, extractions, evidence, conflicts, temporal, clock, archival) {
|
|
1360
|
+
const result = await ctx.models.derive.chat([
|
|
1361
|
+
{ role: 'system', content: VERIFY_SYSTEM },
|
|
1362
|
+
{
|
|
1363
|
+
role: 'user',
|
|
1364
|
+
content: JSON.stringify({
|
|
1365
|
+
mode: page.dream_management,
|
|
1366
|
+
before,
|
|
1367
|
+
after,
|
|
1368
|
+
splits,
|
|
1369
|
+
extracts: extractions.map((extraction) => ({
|
|
1370
|
+
slug: extraction.slug,
|
|
1371
|
+
title: extraction.title,
|
|
1372
|
+
sourceHeading: extraction.sourceHeading,
|
|
1373
|
+
bridge: extraction.bridge,
|
|
1374
|
+
body: extractionPageBody(extraction, page.slug),
|
|
1375
|
+
})),
|
|
1376
|
+
evidence,
|
|
1377
|
+
conflicts,
|
|
1378
|
+
time: temporalPrompt(temporal, clock),
|
|
1379
|
+
archival,
|
|
1380
|
+
}).slice(0, 100_000),
|
|
1381
|
+
},
|
|
1382
|
+
], { schema: VERIFY_SCHEMA, maxTokens: 1_200 });
|
|
1383
|
+
if (!result.ok || !result.value) {
|
|
1384
|
+
return { ok: false, issues: [result.error ?? 'verification failed'], cacheable: false };
|
|
1385
|
+
}
|
|
1386
|
+
const parsed = parseJsonLoose(result.value);
|
|
1387
|
+
if (!parsed ||
|
|
1388
|
+
typeof parsed.ok !== 'boolean' ||
|
|
1389
|
+
!Array.isArray(parsed.issues) ||
|
|
1390
|
+
!parsed.issues.every((issue) => typeof issue === 'string')) {
|
|
1391
|
+
ctx.models.derive.reportInvalidResponse();
|
|
1392
|
+
return { ok: false, issues: ['verifier returned invalid JSON'], cacheable: true };
|
|
1393
|
+
}
|
|
1394
|
+
const issues = parsed.issues.slice(0, 12);
|
|
1395
|
+
return {
|
|
1396
|
+
ok: parsed?.ok === true && issues.length === 0,
|
|
1397
|
+
issues: issues.length ? issues : ['verifier rejected rewrite'],
|
|
1398
|
+
cacheable: true,
|
|
1399
|
+
};
|
|
1400
|
+
}
|
|
1401
|
+
function guardRewrite(input) {
|
|
1402
|
+
const issues = [];
|
|
1403
|
+
const combined = [
|
|
1404
|
+
input.after,
|
|
1405
|
+
...input.splits.map((split) => split.body),
|
|
1406
|
+
...input.extractions.map((extraction) => extractionPageBody(extraction, input.pageSlug)),
|
|
1407
|
+
].join('\n');
|
|
1408
|
+
const beforeItems = itemIds(input.before);
|
|
1409
|
+
const afterItems = itemIds(combined);
|
|
1410
|
+
if (beforeItems.size !== afterItems.size || [...beforeItems].some((id) => !afterItems.has(id))) {
|
|
1411
|
+
issues.push('stable item markers were lost, duplicated or changed');
|
|
1412
|
+
}
|
|
1413
|
+
const missingValues = missingNumericValues(input.before, combined);
|
|
1414
|
+
if (missingValues.length > 0) {
|
|
1415
|
+
const shown = missingValues.slice(0, 12).map((value) => JSON.stringify(value));
|
|
1416
|
+
const remainder = missingValues.length - shown.length;
|
|
1417
|
+
issues.push(`numeric/date/value tokens missing from rewrite: ${shown.join(', ')}` +
|
|
1418
|
+
(remainder > 0 ? ` (+${remainder} more)` : ''));
|
|
1419
|
+
issues.push(...missingValueContexts(input.before, missingValues));
|
|
1420
|
+
}
|
|
1421
|
+
issues.push(...linkIssues(input.before, combined, input.pageSlug, input.splits, input.extractions, input.knownSlugs, input.allowedLinkSlugs));
|
|
1422
|
+
if (input.mode === 'hygiene') {
|
|
1423
|
+
const beforeH1 = firstH1(input.before);
|
|
1424
|
+
const afterH1 = firstH1(input.after);
|
|
1425
|
+
if (beforeH1 !== afterH1)
|
|
1426
|
+
issues.push('the page title/top-level heading changed');
|
|
1427
|
+
const ratio = input.after.length / Math.max(1, input.before.length);
|
|
1428
|
+
if (ratio < 0.6 || ratio > 1.4)
|
|
1429
|
+
issues.push('the hygiene rewrite changed the page size too drastically');
|
|
1430
|
+
if (input.splits.length > 0)
|
|
1431
|
+
issues.push('hygiene pages cannot split');
|
|
1432
|
+
if (input.extractions.length > 0)
|
|
1433
|
+
issues.push('hygiene pages cannot extract');
|
|
1434
|
+
}
|
|
1435
|
+
if (input.splits.length > 0 && input.extractions.length > 0) {
|
|
1436
|
+
issues.push('one curation item cannot split and extract at the same time');
|
|
1437
|
+
}
|
|
1438
|
+
if (input.extractions.length > 0) {
|
|
1439
|
+
issues.push(...extractionAccountingIssues(input.before, input.after, input.extractions, input.incomingAnchors));
|
|
1440
|
+
}
|
|
1441
|
+
if (input.mode === 'synthesize' &&
|
|
1442
|
+
input.after !== input.before &&
|
|
1443
|
+
input.splits.length === 0 &&
|
|
1444
|
+
input.extractions.length === 0 &&
|
|
1445
|
+
!hasMaterialSynthesisChange(input.before, input.after, input.pageSlug)) {
|
|
1446
|
+
issues.push('synthesis rewrite is cosmetic or organizational; no material knowledge was added');
|
|
1447
|
+
}
|
|
1448
|
+
if (input.mode === 'synthesize' && input.conflicts.length > 0 && !/^##\s+Unresolved\s*$/im.test(combined)) {
|
|
1449
|
+
issues.push('known conflicts are not preserved under an Unresolved section');
|
|
1450
|
+
}
|
|
1451
|
+
for (const split of input.splits) {
|
|
1452
|
+
const target = `${input.pageSlug}/${split.suffix}`.toLowerCase();
|
|
1453
|
+
if (input.knownSlugs.has(target))
|
|
1454
|
+
issues.push(`split target already exists: ${target}`);
|
|
1455
|
+
}
|
|
1456
|
+
for (const extraction of input.extractions) {
|
|
1457
|
+
if (input.knownSlugs.has(extraction.slug.toLowerCase())) {
|
|
1458
|
+
issues.push(`extraction target already exists: ${extraction.slug}`);
|
|
1459
|
+
}
|
|
1460
|
+
}
|
|
1461
|
+
if (input.mode === 'synthesize' &&
|
|
1462
|
+
input.conflicts.length === 0 &&
|
|
1463
|
+
!/^##\s+Unresolved\s*$/im.test(input.before) &&
|
|
1464
|
+
/^##\s+Unresolved\s*$/im.test(combined)) {
|
|
1465
|
+
issues.push('an Unresolved section was added even though no unresolved conflict was supplied');
|
|
1466
|
+
}
|
|
1467
|
+
return issues;
|
|
1468
|
+
}
|
|
1469
|
+
/**
|
|
1470
|
+
* Synthesis exists to integrate knowledge, not to spend a high-risk transaction on prose churn.
|
|
1471
|
+
* Headings and Markdown decoration are deliberately ignored: changing "Highlights" to "History
|
|
1472
|
+
* and highlights" must not count as new knowledge. A new evidence-backed wikilink is material by
|
|
1473
|
+
* itself; otherwise the rewrite needs at least two new content terms. The model verifier and
|
|
1474
|
+
* independent curator still decide whether those terms are actually supported and useful.
|
|
1475
|
+
*/
|
|
1476
|
+
function hasMaterialSynthesisChange(before, after, pageSlug) {
|
|
1477
|
+
const priorLinks = linkTargets(before, pageSlug).wiki;
|
|
1478
|
+
const nextLinks = linkTargets(after, pageSlug).wiki;
|
|
1479
|
+
if ([...nextLinks].some((target) => !priorLinks.has(target)))
|
|
1480
|
+
return true;
|
|
1481
|
+
const priorTerms = synthesisTerms(before);
|
|
1482
|
+
const nextTerms = synthesisTerms(after);
|
|
1483
|
+
let added = 0;
|
|
1484
|
+
for (const term of nextTerms) {
|
|
1485
|
+
if (!priorTerms.has(term) && ++added >= 2)
|
|
1486
|
+
return true;
|
|
1487
|
+
}
|
|
1488
|
+
return false;
|
|
1489
|
+
}
|
|
1490
|
+
function synthesisTerms(body) {
|
|
1491
|
+
const prose = body
|
|
1492
|
+
.replace(/<!--[\s\S]*?-->/g, ' ')
|
|
1493
|
+
.split('\n')
|
|
1494
|
+
.filter((line) => !/^\s{0,3}#{1,6}\s+/.test(line))
|
|
1495
|
+
.join('\n')
|
|
1496
|
+
// Link targets are evaluated separately. Keeping them here would let a path rename masquerade
|
|
1497
|
+
// as a factual addition; visible Markdown-link labels remain ordinary prose.
|
|
1498
|
+
.replace(/!\[\[[^\]]+\]\]/g, ' ')
|
|
1499
|
+
.replace(/\[\[[^\]]+\]\]/g, ' ')
|
|
1500
|
+
.replace(/!\[([^\]]*)\]\([^)]*\)/g, '$1')
|
|
1501
|
+
.replace(/\[([^\]]+)\]\([^)]*\)/g, '$1')
|
|
1502
|
+
.normalize('NFKC')
|
|
1503
|
+
.toLowerCase();
|
|
1504
|
+
return new Set(prose.match(/[\p{L}\p{N}]+(?:['’_-][\p{L}\p{N}]+)*/gu) ?? []);
|
|
1505
|
+
}
|
|
1506
|
+
function missingValueContexts(body, values) {
|
|
1507
|
+
const lines = body.split('\n');
|
|
1508
|
+
const contexts = [];
|
|
1509
|
+
for (const value of values.slice(0, 6)) {
|
|
1510
|
+
const index = lines.findIndex((line) => line.includes(value));
|
|
1511
|
+
if (index < 0)
|
|
1512
|
+
continue;
|
|
1513
|
+
const source = lines[index].trim().replace(/\s+/g, ' ');
|
|
1514
|
+
contexts.push(`source body line ${index + 1} for ${JSON.stringify(value)}: ${source.length > 240 ? `${source.slice(0, 237)}...` : source}`);
|
|
1515
|
+
}
|
|
1516
|
+
return contexts;
|
|
1517
|
+
}
|
|
1518
|
+
/** Blank-line cleanup is useful hygiene, but it is not a substantive post-event discovery. */
|
|
1519
|
+
function archiveMeaningKey(body) {
|
|
1520
|
+
return body
|
|
1521
|
+
.replaceAll('\r\n', '\n')
|
|
1522
|
+
.split('\n')
|
|
1523
|
+
.map((line) => line.trimEnd())
|
|
1524
|
+
.filter((line) => line.length > 0)
|
|
1525
|
+
.join('\n');
|
|
1526
|
+
}
|
|
1527
|
+
function linkIssues(before, after, pageSlug, splits, extractions, knownSlugs, allowedLinkSlugs) {
|
|
1528
|
+
const issues = [];
|
|
1529
|
+
const prior = linkTargets(before, pageSlug);
|
|
1530
|
+
const next = linkTargets(after, pageSlug);
|
|
1531
|
+
for (const target of prior.wiki) {
|
|
1532
|
+
if (!next.wiki.has(target))
|
|
1533
|
+
issues.push(`existing wikilink target was removed or changed: [[${target}]]`);
|
|
1534
|
+
}
|
|
1535
|
+
for (const target of prior.markdown) {
|
|
1536
|
+
if (!next.markdown.has(target))
|
|
1537
|
+
issues.push(`existing Markdown link target was removed or changed: ${target}`);
|
|
1538
|
+
}
|
|
1539
|
+
for (const target of next.markdown) {
|
|
1540
|
+
if (prior.markdown.has(target))
|
|
1541
|
+
continue;
|
|
1542
|
+
issues.push(externalLink(target)
|
|
1543
|
+
? `new external URL was invented instead of supplied by evidence: ${target}`
|
|
1544
|
+
: `new internal Markdown link target is not allowed; use an exact wikilink slug: ${target}`);
|
|
1545
|
+
}
|
|
1546
|
+
const proposed = new Set([
|
|
1547
|
+
...splits.map((split) => `${pageSlug}/${split.suffix}`.toLowerCase()),
|
|
1548
|
+
...extractions.map((extraction) => extraction.slug.toLowerCase()),
|
|
1549
|
+
]);
|
|
1550
|
+
for (const target of next.wiki) {
|
|
1551
|
+
if (prior.wiki.has(target))
|
|
1552
|
+
continue;
|
|
1553
|
+
if (!knownSlugs.has(target) && !proposed.has(target)) {
|
|
1554
|
+
issues.push(`new wikilink does not resolve to an existing or proposed page: [[${target}]]`);
|
|
1555
|
+
}
|
|
1556
|
+
else if (!allowedLinkSlugs.has(target) && !proposed.has(target) && target !== pageSlug.toLowerCase()) {
|
|
1557
|
+
issues.push(`new wikilink target was not supplied by the evidence graph: [[${target}]]`);
|
|
1558
|
+
}
|
|
1559
|
+
}
|
|
1560
|
+
return issues;
|
|
1561
|
+
}
|
|
1562
|
+
function linkTargets(body, pageSlug) {
|
|
1563
|
+
const wiki = new Set();
|
|
1564
|
+
const markdown = new Set();
|
|
1565
|
+
for (const match of body.matchAll(/\[\[([^\]|#]+)(?:[#|][^\]]*)?\]\]/g)) {
|
|
1566
|
+
wiki.add(normalizeLinkTarget(match[1]).toLowerCase());
|
|
1567
|
+
}
|
|
1568
|
+
for (const match of body.matchAll(/(?<!!)\[[^\]]*\]\(\s*<?([^)\s>]+)>?(?:\s+[^)]*)?\)/g)) {
|
|
1569
|
+
const target = match[1];
|
|
1570
|
+
markdown.add(externalLink(target) || target.startsWith('#') ? target : normalizeLinkTarget(target, pageSlug));
|
|
1571
|
+
}
|
|
1572
|
+
return { wiki, markdown };
|
|
1573
|
+
}
|
|
1574
|
+
function externalLink(target) {
|
|
1575
|
+
return /^(?:[a-z][a-z0-9+.-]*:|\/\/)/i.test(target);
|
|
1576
|
+
}
|
|
1577
|
+
export function linkIssuesForTesting(before, after, pageSlug, knownSlugs) {
|
|
1578
|
+
const known = new Set(knownSlugs.map((slug) => slug.toLowerCase()));
|
|
1579
|
+
return linkIssues(before, after, pageSlug, [], [], known, known);
|
|
1580
|
+
}
|
|
1581
|
+
function renderEvidence(evidence) {
|
|
1582
|
+
return evidence.map((row) => {
|
|
1583
|
+
const summary = row.relationship === 'about' && row.summary ? ` — ${row.summary}` : '';
|
|
1584
|
+
const heading = `[[${row.slug}]] (${row.relationship})${summary}`;
|
|
1585
|
+
const details = [
|
|
1586
|
+
...row.facts.map((fact) => `- ${fact.claim}`),
|
|
1587
|
+
...row.events.map((event) => `- ${event.date}: ${event.summary}`),
|
|
1588
|
+
];
|
|
1589
|
+
return details.length ? `${heading}\n${details.join('\n')}` : heading;
|
|
1590
|
+
});
|
|
1591
|
+
}
|
|
1592
|
+
function renderConflicts(conflicts) {
|
|
1593
|
+
return conflicts.map((row) => `${row.subject} / ${row.attribute}: ${row.claim} [[${row.slug}]]`);
|
|
1594
|
+
}
|
|
1595
|
+
function temporalForRow(page) {
|
|
1596
|
+
try {
|
|
1597
|
+
const frontmatter = JSON.parse(page.frontmatter);
|
|
1598
|
+
if (!frontmatter || typeof frontmatter !== 'object' || Array.isArray(frontmatter))
|
|
1599
|
+
return null;
|
|
1600
|
+
return readTemporalDeclaration(frontmatter).metadata;
|
|
1601
|
+
}
|
|
1602
|
+
catch {
|
|
1603
|
+
return null;
|
|
1604
|
+
}
|
|
1605
|
+
}
|
|
1606
|
+
function temporalResult(metadata, source, clock, archival) {
|
|
1607
|
+
return metadata && source
|
|
1608
|
+
? {
|
|
1609
|
+
temporal: {
|
|
1610
|
+
source,
|
|
1611
|
+
state: temporalState(metadata, clock),
|
|
1612
|
+
until: metadata.until,
|
|
1613
|
+
archival,
|
|
1614
|
+
},
|
|
1615
|
+
}
|
|
1616
|
+
: {};
|
|
1617
|
+
}
|
|
1618
|
+
function curateInputHash(page, evidence, conflicts, temporal, eventState, extractionPolicyHash, incomingLinksFingerprint) {
|
|
1619
|
+
return sha256(JSON.stringify({
|
|
1620
|
+
version: CURATE_FINGERPRINT_VERSION,
|
|
1621
|
+
page: {
|
|
1622
|
+
slug: page.slug,
|
|
1623
|
+
title: page.title,
|
|
1624
|
+
role: page.role,
|
|
1625
|
+
mode: page.dream_management,
|
|
1626
|
+
about: page.about,
|
|
1627
|
+
frontmatter: page.frontmatter,
|
|
1628
|
+
bodyHash: page.body_hash,
|
|
1629
|
+
},
|
|
1630
|
+
// Hygiene deliberately has empty arrays here: its authority is confined to this page.
|
|
1631
|
+
evidence: evidence.map((row) => ({
|
|
1632
|
+
slug: row.slug,
|
|
1633
|
+
summary: row.relationship === 'about' ? row.summary : null,
|
|
1634
|
+
about: row.about,
|
|
1635
|
+
role: row.role,
|
|
1636
|
+
bodyHash: row.body_hash,
|
|
1637
|
+
relationship: row.relationship,
|
|
1638
|
+
facts: row.facts,
|
|
1639
|
+
events: row.events,
|
|
1640
|
+
})),
|
|
1641
|
+
conflicts,
|
|
1642
|
+
temporal,
|
|
1643
|
+
// Unlike the current date, this changes only once for a bounded event. It schedules one
|
|
1644
|
+
// archival assessment without making every page stale every day.
|
|
1645
|
+
eventState,
|
|
1646
|
+
extractionPolicyHash: page.dream_management === 'synthesize' ? extractionPolicyHash : null,
|
|
1647
|
+
incomingLinksFingerprint: page.dream_management === 'synthesize' ? incomingLinksFingerprint : null,
|
|
1648
|
+
}));
|
|
1649
|
+
}
|
|
1650
|
+
function incomingLinkFingerprint(ctx, pageId) {
|
|
1651
|
+
const rows = ctx.store.db
|
|
1652
|
+
.prepare(`SELECT p.slug, p.body_hash, l.kind, l.line FROM links l
|
|
1653
|
+
JOIN pages p ON p.id = l.from_page
|
|
1654
|
+
WHERE l.to_page = ? AND l.from_page != ?
|
|
1655
|
+
ORDER BY p.slug, l.kind, l.line`)
|
|
1656
|
+
.all(pageId, pageId);
|
|
1657
|
+
return sha256(JSON.stringify(rows));
|
|
1658
|
+
}
|
|
1659
|
+
function curationDue(page, inputHash, dryRun, includePreviewed) {
|
|
1660
|
+
if (page.curate_input_hash !== inputHash)
|
|
1661
|
+
return true;
|
|
1662
|
+
if (includePreviewed && page.curate_status === 'preview')
|
|
1663
|
+
return true;
|
|
1664
|
+
// A write-enabled pass must rerun a previously accepted preview once. Rejected and unchanged
|
|
1665
|
+
// inputs are already complete decisions, and applied input is current by definition.
|
|
1666
|
+
return !dryRun && page.curate_status === 'preview';
|
|
1667
|
+
}
|
|
1668
|
+
/** Mark successfully plan-applied pages against their post-write fingerprints. */
|
|
1669
|
+
export function markCurateApplied(ctx, slugs) {
|
|
1670
|
+
const state = new Map();
|
|
1671
|
+
const clock = temporalClock();
|
|
1672
|
+
const extractionPolicyHash = extractionPolicyFingerprint(ctx);
|
|
1673
|
+
for (const slug of slugs) {
|
|
1674
|
+
const refreshed = pageForSlug(ctx, slug);
|
|
1675
|
+
if (!refreshed)
|
|
1676
|
+
continue;
|
|
1677
|
+
const temporal = temporalForRow(refreshed);
|
|
1678
|
+
const eventState = temporal ? temporalState(temporal, clock) : null;
|
|
1679
|
+
const archival = refreshed.dream_management === 'synthesize' && eventState === 'past';
|
|
1680
|
+
const allEvidence = refreshed.dream_management === 'synthesize' ? evidenceFor(ctx, refreshed) : [];
|
|
1681
|
+
const evidence = archival ? archivalEvidence(allEvidence) : allEvidence;
|
|
1682
|
+
const conflicts = refreshed.dream_management === 'synthesize' ? conflictsFor(ctx, refreshed.id) : [];
|
|
1683
|
+
queueCurateState(state, refreshed.id, curateInputHash(refreshed, evidence, conflicts, temporal, eventState, extractionPolicyHash, incomingLinkFingerprint(ctx, refreshed.id)), 'applied');
|
|
1684
|
+
}
|
|
1685
|
+
persistCurateState(ctx, state.values());
|
|
1686
|
+
}
|
|
1687
|
+
/** Cache a completed plan decision so the same rejected input is not proposed every cycle. */
|
|
1688
|
+
export function markCurateRejected(ctx, pages) {
|
|
1689
|
+
const state = new Map();
|
|
1690
|
+
for (const page of pages) {
|
|
1691
|
+
const row = pageForSlug(ctx, page.slug);
|
|
1692
|
+
if (row)
|
|
1693
|
+
queueCurateState(state, row.id, page.inputHash, 'rejected');
|
|
1694
|
+
}
|
|
1695
|
+
persistCurateState(ctx, state.values());
|
|
1696
|
+
}
|
|
1697
|
+
function queueCurateState(state, pageId, inputHash, status) {
|
|
1698
|
+
state.set(pageId, { pageId, inputHash, status });
|
|
1699
|
+
}
|
|
1700
|
+
function persistCurateState(ctx, values) {
|
|
1701
|
+
const rows = [...values];
|
|
1702
|
+
if (rows.length === 0)
|
|
1703
|
+
return;
|
|
1704
|
+
const update = ctx.store.db.prepare(`UPDATE pages SET curate_input_hash = ?, curate_status = ?, curated_at = ? WHERE id = ?`);
|
|
1705
|
+
const now = new Date().toISOString();
|
|
1706
|
+
ctx.store.transaction(() => {
|
|
1707
|
+
for (const row of rows)
|
|
1708
|
+
update.run(row.inputHash, row.status, now, row.pageId);
|
|
1709
|
+
});
|
|
1710
|
+
}
|
|
1711
|
+
function pageForSlug(ctx, slug) {
|
|
1712
|
+
return (ctx.store.db
|
|
1713
|
+
.prepare(`SELECT id, slug, rel_path, title, role, dream_management, about, frontmatter, aliases, body_hash, bytes,
|
|
1714
|
+
curate_input_hash, curate_status
|
|
1715
|
+
FROM pages WHERE slug = ? AND role = 'knowledge'
|
|
1716
|
+
AND dream_management IN ('hygiene', 'synthesize')`)
|
|
1717
|
+
.get(slug) ?? null);
|
|
1718
|
+
}
|
|
1719
|
+
function itemIds(text) {
|
|
1720
|
+
const out = new Set();
|
|
1721
|
+
for (const line of text.split('\n')) {
|
|
1722
|
+
const match = AKNO_ITEM.exec(line);
|
|
1723
|
+
if (match) {
|
|
1724
|
+
const id = match[1].trim().split(/\s+/)[0];
|
|
1725
|
+
if (out.has(id))
|
|
1726
|
+
out.add(`duplicate:${id}`);
|
|
1727
|
+
else
|
|
1728
|
+
out.add(id);
|
|
1729
|
+
}
|
|
1730
|
+
}
|
|
1731
|
+
return out;
|
|
1732
|
+
}
|
|
1733
|
+
function firstH1(body) {
|
|
1734
|
+
return (body
|
|
1735
|
+
.split('\n')
|
|
1736
|
+
.map((line) => /^#\s+(.+?)\s*$/.exec(line)?.[1] ?? null)
|
|
1737
|
+
.find(Boolean) ?? null);
|
|
1738
|
+
}
|
|
1739
|
+
function cleanSplits(value, limit, minBytes) {
|
|
1740
|
+
if (!Array.isArray(value) || limit <= 0)
|
|
1741
|
+
return [];
|
|
1742
|
+
const out = [];
|
|
1743
|
+
for (const entry of value) {
|
|
1744
|
+
if (!entry || typeof entry !== 'object')
|
|
1745
|
+
continue;
|
|
1746
|
+
const row = entry;
|
|
1747
|
+
const suffix = typeof row.suffix === 'string' ? row.suffix.trim().toLowerCase() : '';
|
|
1748
|
+
const title = typeof row.title === 'string' ? row.title.trim() : '';
|
|
1749
|
+
const body = typeof row.body === 'string' ? endWithNewline(row.body) : '';
|
|
1750
|
+
if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(suffix) || !title || Buffer.byteLength(body) < minBytes)
|
|
1751
|
+
continue;
|
|
1752
|
+
out.push({ suffix, title, body });
|
|
1753
|
+
if (out.length >= limit)
|
|
1754
|
+
break;
|
|
1755
|
+
}
|
|
1756
|
+
return out;
|
|
1757
|
+
}
|
|
1758
|
+
function allowedExtractionFolders(ctx) {
|
|
1759
|
+
return (folderCatalog(ctx.config, ctx.store)
|
|
1760
|
+
.filter((entry) => entry.eligible &&
|
|
1761
|
+
entry.role === 'knowledge' &&
|
|
1762
|
+
entry.remember === 'integrate' &&
|
|
1763
|
+
entry.path.length > 0)
|
|
1764
|
+
// A bounded taxonomy keeps a large knowledge base from crowding the page and evidence out of
|
|
1765
|
+
// the model context. The same exact list is used by the deterministic destination guard.
|
|
1766
|
+
.slice(0, 120));
|
|
1767
|
+
}
|
|
1768
|
+
function extractionPolicyFingerprint(ctx, folders = allowedExtractionFolders(ctx)) {
|
|
1769
|
+
const settings = ctx.config.maintenance.curate;
|
|
1770
|
+
return sha256(JSON.stringify({
|
|
1771
|
+
folders,
|
|
1772
|
+
maxExtracts: settings.maxExtracts,
|
|
1773
|
+
extractAfterBytes: settings.extractAfterBytes,
|
|
1774
|
+
extractSectionBytes: settings.extractSectionBytes,
|
|
1775
|
+
policies: ctx.config.maintenance.policies,
|
|
1776
|
+
}));
|
|
1777
|
+
}
|
|
1778
|
+
/** Re-check a sealed extraction against the current user-owned taxonomy before any write. */
|
|
1779
|
+
export function extractionDestinationIssues(ctx, sourceSlug, targetSlug) {
|
|
1780
|
+
const slash = targetSlug.lastIndexOf('/');
|
|
1781
|
+
const parent = slash > 0 ? targetSlug.slice(0, slash) : '';
|
|
1782
|
+
const allowed = allowedExtractionFolders(ctx).some((entry) => entry.path.toLowerCase() === parent.toLowerCase());
|
|
1783
|
+
const issues = allowed
|
|
1784
|
+
? destinationRuleIssues(targetSlug, ctx.config.rules)
|
|
1785
|
+
: [`extraction destination is no longer an allowed knowledge folder: ${parent || '(root)'}`];
|
|
1786
|
+
if (targetSlug.toLowerCase().startsWith(`${sourceSlug.toLowerCase()}/`)) {
|
|
1787
|
+
issues.push('extraction destination became a child of the source; use a split instead');
|
|
1788
|
+
}
|
|
1789
|
+
return issues;
|
|
1790
|
+
}
|
|
1791
|
+
function extractionPrompt(folders, sections) {
|
|
1792
|
+
if (folders.length === 0 || sections.length === 0) {
|
|
1793
|
+
return '\n\nNo exact extraction section and destination are available for this item. Return "extracts": [].';
|
|
1794
|
+
}
|
|
1795
|
+
return ('\n\nAllowed extraction destination folders (use one exact path as the parent):\n' +
|
|
1796
|
+
JSON.stringify(folders.map((entry) => ({
|
|
1797
|
+
path: entry.path,
|
|
1798
|
+
...(entry.description ? { purpose: entry.description } : {}),
|
|
1799
|
+
}))) +
|
|
1800
|
+
'\nEligible extraction sections (copy one exact heading into source_heading):\n' +
|
|
1801
|
+
JSON.stringify(sections.map((section) => ({
|
|
1802
|
+
source_heading: section.heading,
|
|
1803
|
+
bytes: Buffer.byteLength(section.body),
|
|
1804
|
+
}))));
|
|
1805
|
+
}
|
|
1806
|
+
function extractionSections(sourceBody, minBytes) {
|
|
1807
|
+
const newline = sourceBody.includes('\r\n') ? '\r\n' : '\n';
|
|
1808
|
+
const lines = sourceBody.split(newline);
|
|
1809
|
+
const headings = lines.flatMap((line, index) => {
|
|
1810
|
+
const match = /^(\s{0,3})(#{2,6})\s+(.+?)\s*#*\s*$/.exec(line);
|
|
1811
|
+
return match ? [{ heading: line.trimEnd(), level: match[2].length, index }] : [];
|
|
1812
|
+
});
|
|
1813
|
+
const counts = new Map();
|
|
1814
|
+
for (const heading of headings)
|
|
1815
|
+
counts.set(heading.heading, (counts.get(heading.heading) ?? 0) + 1);
|
|
1816
|
+
const sections = [];
|
|
1817
|
+
for (const [position, heading] of headings.entries()) {
|
|
1818
|
+
if (counts.get(heading.heading) !== 1)
|
|
1819
|
+
continue;
|
|
1820
|
+
const next = headings.slice(position + 1).find((candidate) => candidate.level <= heading.level);
|
|
1821
|
+
const endIndex = next?.index ?? lines.length;
|
|
1822
|
+
const body = `${lines.slice(heading.index, endIndex).join(newline).trimEnd()}${newline}`;
|
|
1823
|
+
if (Buffer.byteLength(body) < minBytes)
|
|
1824
|
+
continue;
|
|
1825
|
+
sections.push({
|
|
1826
|
+
heading: heading.heading,
|
|
1827
|
+
body,
|
|
1828
|
+
startIndex: heading.index,
|
|
1829
|
+
endIndex,
|
|
1830
|
+
});
|
|
1831
|
+
}
|
|
1832
|
+
return sections;
|
|
1833
|
+
}
|
|
1834
|
+
function cleanExtractions(value, options) {
|
|
1835
|
+
if (!Array.isArray(value) || value.length === 0)
|
|
1836
|
+
return { extractions: [], issues: [] };
|
|
1837
|
+
if (!options.available || options.limit <= 0) {
|
|
1838
|
+
return { extractions: [], issues: ['an extraction was proposed when no extraction slot was available'] };
|
|
1839
|
+
}
|
|
1840
|
+
if (value.length > options.limit) {
|
|
1841
|
+
return { extractions: [], issues: ['a page may propose at most one extraction'] };
|
|
1842
|
+
}
|
|
1843
|
+
const entry = value[0];
|
|
1844
|
+
if (!entry || typeof entry !== 'object') {
|
|
1845
|
+
return { extractions: [], issues: ['the extraction proposal is malformed'] };
|
|
1846
|
+
}
|
|
1847
|
+
const row = entry;
|
|
1848
|
+
const proposedSlug = typeof row.slug === 'string' ? row.slug.trim().replace(/^\/+|\/+$/g, '') : '';
|
|
1849
|
+
const title = typeof row.title === 'string' ? row.title.trim() : '';
|
|
1850
|
+
const sourceHeading = typeof row.source_heading === 'string' ? row.source_heading.trimEnd() : '';
|
|
1851
|
+
const section = extractionSections(options.sourceBody, options.minBytes).find((candidate) => candidate.heading === sourceHeading);
|
|
1852
|
+
const bridge = typeof row.bridge === 'string' ? row.bridge.trim() : '';
|
|
1853
|
+
const slash = proposedSlug.lastIndexOf('/');
|
|
1854
|
+
const proposedFolder = slash > 0 ? proposedSlug.slice(0, slash) : '';
|
|
1855
|
+
const basename = slash > 0 ? proposedSlug.slice(slash + 1) : '';
|
|
1856
|
+
const folder = options.folders.find((candidate) => candidate.path.toLowerCase() === proposedFolder.toLowerCase());
|
|
1857
|
+
const slug = folder ? `${folder.path}/${basename}` : proposedSlug;
|
|
1858
|
+
const issues = [];
|
|
1859
|
+
if (!folder)
|
|
1860
|
+
issues.push(`extraction destination is not an allowed knowledge folder: ${proposedFolder || '(root)'}`);
|
|
1861
|
+
if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/.test(basename)) {
|
|
1862
|
+
issues.push('extraction destination basename must be lowercase and hyphenated');
|
|
1863
|
+
}
|
|
1864
|
+
if (!title)
|
|
1865
|
+
issues.push('extraction title is empty');
|
|
1866
|
+
if (!section)
|
|
1867
|
+
issues.push('extraction source_heading is not one exact eligible source section');
|
|
1868
|
+
if (!bridge || bridge.includes('\n') || bridge.includes('<!--') || bridge.includes('-->')) {
|
|
1869
|
+
issues.push('extraction bridge must be one short plain Markdown paragraph');
|
|
1870
|
+
}
|
|
1871
|
+
else if (Buffer.byteLength(bridge) > 400) {
|
|
1872
|
+
issues.push('extraction bridge is too long');
|
|
1873
|
+
}
|
|
1874
|
+
if (slug.toLowerCase().startsWith(`${options.sourceSlug.toLowerCase()}/`)) {
|
|
1875
|
+
issues.push('extraction destination is a child of the source; use a split instead');
|
|
1876
|
+
}
|
|
1877
|
+
if (options.knownSlugs.has(slug.toLowerCase()))
|
|
1878
|
+
issues.push(`extraction target already exists: ${slug}`);
|
|
1879
|
+
if (folder && basename)
|
|
1880
|
+
issues.push(...destinationRuleIssues(slug, options.rules));
|
|
1881
|
+
return issues.length > 0
|
|
1882
|
+
? { extractions: [], issues }
|
|
1883
|
+
: {
|
|
1884
|
+
extractions: [
|
|
1885
|
+
{
|
|
1886
|
+
slug,
|
|
1887
|
+
title,
|
|
1888
|
+
body: section.body,
|
|
1889
|
+
bridge,
|
|
1890
|
+
sourceHeading,
|
|
1891
|
+
startIndex: section.startIndex,
|
|
1892
|
+
endIndex: section.endIndex,
|
|
1893
|
+
},
|
|
1894
|
+
],
|
|
1895
|
+
issues: [],
|
|
1896
|
+
};
|
|
1897
|
+
}
|
|
1898
|
+
function destinationRuleIssues(slug, rules) {
|
|
1899
|
+
const issues = [];
|
|
1900
|
+
const rule = effectiveRule(slug, rules);
|
|
1901
|
+
const role = rule.role ?? 'knowledge';
|
|
1902
|
+
const remember = rule.remember ?? 'deny';
|
|
1903
|
+
if (role !== 'knowledge' || remember !== 'integrate') {
|
|
1904
|
+
issues.push(`extraction destination is not opted-in integrated knowledge: ${slug}`);
|
|
1905
|
+
}
|
|
1906
|
+
const basename = slug.slice(slug.lastIndexOf('/') + 1);
|
|
1907
|
+
if (rule.slug_pattern) {
|
|
1908
|
+
try {
|
|
1909
|
+
if (!new RegExp(rule.slug_pattern).test(basename)) {
|
|
1910
|
+
issues.push(`extraction destination does not satisfy its folder slug pattern: ${slug}`);
|
|
1911
|
+
}
|
|
1912
|
+
}
|
|
1913
|
+
catch {
|
|
1914
|
+
issues.push(`extraction destination folder has an invalid slug pattern: ${slug}`);
|
|
1915
|
+
}
|
|
1916
|
+
}
|
|
1917
|
+
const depthRule = rules.find((candidate) => candidate.max_depth !== undefined && matchesGlob(slug, candidate.glob));
|
|
1918
|
+
if (depthRule?.max_depth !== undefined) {
|
|
1919
|
+
const baseDepth = depthRule.glob
|
|
1920
|
+
.replace(/\/\*\*?$/, '')
|
|
1921
|
+
.split('/')
|
|
1922
|
+
.filter(Boolean).length;
|
|
1923
|
+
const depth = slug.split('/').length - baseDepth;
|
|
1924
|
+
if (depth > depthRule.max_depth) {
|
|
1925
|
+
issues.push(`extraction destination exceeds its folder depth limit: ${slug}`);
|
|
1926
|
+
}
|
|
1927
|
+
}
|
|
1928
|
+
return issues;
|
|
1929
|
+
}
|
|
1930
|
+
function withExtractionBridge(body, extraction) {
|
|
1931
|
+
const newline = body.includes('\r\n') ? '\r\n' : '\n';
|
|
1932
|
+
const lines = body.split(newline);
|
|
1933
|
+
const managed = `<!-- akno:extract target=${JSON.stringify(extraction.slug)} -->${newline}` +
|
|
1934
|
+
`${extraction.bridge}${newline}` +
|
|
1935
|
+
`<!-- /akno:extract -->${newline}`;
|
|
1936
|
+
return [...lines.slice(0, extraction.startIndex), managed, ...lines.slice(extraction.endIndex)].join(newline);
|
|
1937
|
+
}
|
|
1938
|
+
function extractionAccountingIssues(before, after, extractions, incomingAnchors) {
|
|
1939
|
+
const issues = [];
|
|
1940
|
+
const prior = nonBlankLineCounts(before);
|
|
1941
|
+
const retained = nonBlankLineCounts(after);
|
|
1942
|
+
const moved = nonBlankLineCounts(extractions.map((entry) => entry.body).join('\n'));
|
|
1943
|
+
const combined = nonBlankLineCounts([after, ...extractions.map((entry) => entry.body)].join('\n'));
|
|
1944
|
+
for (const [line, count] of prior) {
|
|
1945
|
+
if ((combined.get(line) ?? 0) !== count) {
|
|
1946
|
+
issues.push('extraction did not account for every source line exactly once');
|
|
1947
|
+
break;
|
|
1948
|
+
}
|
|
1949
|
+
}
|
|
1950
|
+
for (const [line, count] of moved) {
|
|
1951
|
+
if ((prior.get(line) ?? 0) < count) {
|
|
1952
|
+
issues.push('extraction body contains authored text that was not copied verbatim from the source');
|
|
1953
|
+
break;
|
|
1954
|
+
}
|
|
1955
|
+
}
|
|
1956
|
+
const priorCount = [...prior.values()].reduce((total, count) => total + count, 0);
|
|
1957
|
+
const retainedCount = [...prior].reduce((total, [line, count]) => total + Math.min(count, retained.get(line) ?? 0), 0);
|
|
1958
|
+
if (moved.size === 0 || retainedCount === priorCount) {
|
|
1959
|
+
issues.push('extraction did not move any authored source lines');
|
|
1960
|
+
}
|
|
1961
|
+
if (retainedCount < 2 || retainedCount / Math.max(1, priorCount) < 0.25) {
|
|
1962
|
+
issues.push('source page does not retain enough of its original purpose after extraction');
|
|
1963
|
+
}
|
|
1964
|
+
if (firstH1(before) !== firstH1(after)) {
|
|
1965
|
+
issues.push('source page heading changed during extraction');
|
|
1966
|
+
}
|
|
1967
|
+
const movedHeadings = headingReferences(extractions.map((entry) => entry.body).join('\n'));
|
|
1968
|
+
if ([...incomingAnchors].some((anchor) => movedHeadings.has(anchor))) {
|
|
1969
|
+
issues.push('an incoming link targets a heading that the extraction would move');
|
|
1970
|
+
}
|
|
1971
|
+
for (const extraction of extractions) {
|
|
1972
|
+
const occurrences = after.split(extraction.bridge).length - 1;
|
|
1973
|
+
if (occurrences !== 1)
|
|
1974
|
+
issues.push('extraction bridge must appear exactly once in the source');
|
|
1975
|
+
if (!linkTargets(extraction.bridge, '').wiki.has(extraction.slug.toLowerCase())) {
|
|
1976
|
+
issues.push(`extraction bridge does not link to its exact destination: [[${extraction.slug}]]`);
|
|
1977
|
+
}
|
|
1978
|
+
}
|
|
1979
|
+
return [...new Set(issues)];
|
|
1980
|
+
}
|
|
1981
|
+
function nonBlankLineCounts(text) {
|
|
1982
|
+
const counts = new Map();
|
|
1983
|
+
for (const raw of text.replaceAll('\r\n', '\n').split('\n')) {
|
|
1984
|
+
const line = raw.trimEnd();
|
|
1985
|
+
if (!line.trim())
|
|
1986
|
+
continue;
|
|
1987
|
+
counts.set(line, (counts.get(line) ?? 0) + 1);
|
|
1988
|
+
}
|
|
1989
|
+
return counts;
|
|
1990
|
+
}
|
|
1991
|
+
async function incomingHeadingAnchors(ctx, sourcePageId, sourceSlug) {
|
|
1992
|
+
const rows = ctx.store.db
|
|
1993
|
+
.prepare(`SELECT p.slug, p.rel_path, l.line FROM links l
|
|
1994
|
+
JOIN pages p ON p.id = l.from_page
|
|
1995
|
+
WHERE (l.to_page = ? OR lower(l.to_slug) = lower(?)) AND l.from_page != ?
|
|
1996
|
+
ORDER BY p.slug, l.line`)
|
|
1997
|
+
.all(sourcePageId, sourceSlug, sourcePageId);
|
|
1998
|
+
const anchors = new Set();
|
|
1999
|
+
for (const row of rows) {
|
|
2000
|
+
const content = await fsp
|
|
2001
|
+
.readFile(path.join(ctx.config.aknoPath, row.rel_path), 'utf8')
|
|
2002
|
+
.catch(() => null);
|
|
2003
|
+
if (content === null)
|
|
2004
|
+
continue;
|
|
2005
|
+
const line = content.replaceAll('\r\n', '\n').split('\n')[row.line - 1] ?? '';
|
|
2006
|
+
for (const match of line.matchAll(/\[\[([^\]|#]+)#([^\]|]+)(?:\|[^\]]*)?\]\]/g)) {
|
|
2007
|
+
if (normalizeLinkTarget(match[1]).toLowerCase() === sourceSlug.toLowerCase()) {
|
|
2008
|
+
anchors.add(normalizeHeadingReference(match[2]));
|
|
2009
|
+
}
|
|
2010
|
+
}
|
|
2011
|
+
for (const match of line.matchAll(/(?<!!)\[[^\]]*\]\(\s*<?([^\s)>]+)>?(?:\s+[^)]*)?\)/g)) {
|
|
2012
|
+
const href = match[1];
|
|
2013
|
+
const hash = href.indexOf('#');
|
|
2014
|
+
if (hash < 0)
|
|
2015
|
+
continue;
|
|
2016
|
+
const target = href.slice(0, hash);
|
|
2017
|
+
if (normalizeLinkTarget(target, row.slug).toLowerCase() === sourceSlug.toLowerCase()) {
|
|
2018
|
+
anchors.add(normalizeHeadingReference(href.slice(hash + 1)));
|
|
2019
|
+
}
|
|
2020
|
+
}
|
|
2021
|
+
}
|
|
2022
|
+
return anchors;
|
|
2023
|
+
}
|
|
2024
|
+
/** Re-check heading-fragment safety against current backlinks at plan apply and verification time. */
|
|
2025
|
+
export async function extractionIncomingHeadingIssues(ctx, sourceSlug, extractedBody) {
|
|
2026
|
+
const source = ctx.store.db.prepare('SELECT id FROM pages WHERE slug = ?').get(sourceSlug);
|
|
2027
|
+
if (!source)
|
|
2028
|
+
return [`the extraction source is missing from the structural index: ${sourceSlug}`];
|
|
2029
|
+
const incoming = await incomingHeadingAnchors(ctx, source.id, sourceSlug);
|
|
2030
|
+
const moved = headingReferences(extractedBody);
|
|
2031
|
+
return [...incoming].some((anchor) => moved.has(anchor))
|
|
2032
|
+
? ['an incoming link targets a heading that the extraction would move']
|
|
2033
|
+
: [];
|
|
2034
|
+
}
|
|
2035
|
+
function headingReferences(body) {
|
|
2036
|
+
return new Set(body
|
|
2037
|
+
.split('\n')
|
|
2038
|
+
.map((line) => /^\s{0,3}#{1,6}\s+(.+?)\s*#*\s*$/.exec(line)?.[1] ?? null)
|
|
2039
|
+
.filter((heading) => heading !== null)
|
|
2040
|
+
.map(normalizeHeadingReference));
|
|
2041
|
+
}
|
|
2042
|
+
function normalizeHeadingReference(value) {
|
|
2043
|
+
let decoded = value;
|
|
2044
|
+
try {
|
|
2045
|
+
decoded = decodeURIComponent(value);
|
|
2046
|
+
}
|
|
2047
|
+
catch {
|
|
2048
|
+
// A malformed escape should not take maintenance down; compare its literal form instead.
|
|
2049
|
+
}
|
|
2050
|
+
return decoded
|
|
2051
|
+
.normalize('NFKC')
|
|
2052
|
+
.toLowerCase()
|
|
2053
|
+
.replace(/[^\p{L}\p{N}]+/gu, '');
|
|
2054
|
+
}
|
|
2055
|
+
function childPage(split, canonicalSlug) {
|
|
2056
|
+
return `---\ntitle: ${JSON.stringify(split.title)}\nakno:\n role: knowledge\n management:\n remember: integrate\n dream: synthesize\n about:\n - ${JSON.stringify(canonicalSlug)}\n---\n\n${split.body}`;
|
|
2057
|
+
}
|
|
2058
|
+
function extractionPage(extraction, sourceSlug) {
|
|
2059
|
+
return `---\ntitle: ${JSON.stringify(extraction.title)}\nakno:\n role: knowledge\n management:\n remember: integrate\n dream: synthesize\n---\n\n${extractionPageBody(extraction, sourceSlug)}`;
|
|
2060
|
+
}
|
|
2061
|
+
function extractionPageBody(extraction, sourceSlug) {
|
|
2062
|
+
return (`${extraction.body.trimEnd()}\n\n` +
|
|
2063
|
+
`<!-- akno:extracted-from source=${JSON.stringify(sourceSlug)} -->\n` +
|
|
2064
|
+
`Extracted from [[${sourceSlug}]].\n` +
|
|
2065
|
+
'<!-- /akno:extracted-from -->\n');
|
|
2066
|
+
}
|
|
2067
|
+
function endWithNewline(text) {
|
|
2068
|
+
return text.endsWith('\n') ? text : `${text}\n`;
|
|
2069
|
+
}
|
|
2070
|
+
//# sourceMappingURL=curate.js.map
|