mlx-bun 0.5.0 → 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1449 -109
- package/THIRD_PARTY_NOTICES.md +882 -0
- package/bin/mlx-bun.convert.mjs +4 -0
- package/bin/mlx-bun.fuse.mjs +4 -0
- package/bin/mlx-bun.generate.mjs +4 -0
- package/bin/mlx-bun.lora.mjs +4 -0
- package/bin/mlx-bun.mjs +13 -21
- package/bin/mlx-bun.server.mjs +4 -0
- package/bin/mlx-bun.upload.mjs +4 -0
- package/dist/web/app.js +9584 -0
- package/package.json +50 -53
- package/src/cli/ane-install.ts +76 -0
- package/src/cli/args.ts +179 -0
- package/src/cli/engine-entry.ts +68 -0
- package/src/cli/inference.ts +214 -0
- package/src/cli/job-entry.ts +97 -0
- package/src/cli/library-host.ts +117 -0
- package/src/cli/main.ts +57 -0
- package/src/cli/memory-engine.ts +163 -0
- package/src/cli/memory.ts +36 -0
- package/src/cli/mlx-lm-aliases.ts +309 -0
- package/src/cli/model-choice.ts +33 -0
- package/src/cli/model-selection.ts +84 -0
- package/src/cli/module-host.ts +11 -0
- package/src/cli/module-verbs.ts +50 -0
- package/src/cli/numerical-policy.ts +12 -0
- package/src/cli/perplexity.ts +89 -0
- package/src/cli/serve-host.ts +423 -0
- package/src/cli/serve-isolated.ts +217 -0
- package/src/cli/serve-options.ts +106 -0
- package/src/cli/serve-state.ts +205 -0
- package/src/cli/serve.ts +500 -0
- package/src/cli/served-model-host.ts +102 -0
- package/src/cli/server-entry.ts +148 -0
- package/src/cli/serving-unit.ts +177 -0
- package/src/{tui.ts → cli/terminal.ts} +13 -8
- package/src/cli/worker-adapters.ts +34 -0
- package/src/cli/worker-entry.ts +272 -0
- package/src/cli/worker-unit.ts +179 -0
- package/src/engine/adapter-operation.ts +26 -0
- package/src/engine/cache-services.ts +114 -0
- package/src/engine/completion.ts +39 -0
- package/src/engine/generation-gateway.ts +358 -0
- package/src/engine/index.ts +59 -0
- package/src/engine/kv-budget.ts +120 -0
- package/src/engine/media-preparation.ts +232 -0
- package/src/engine/model-binding.ts +24 -0
- package/src/engine/model-host.ts +513 -0
- package/src/engine/model-serving.ts +89 -0
- package/src/{serve → engine}/preparation.ts +7 -5
- package/src/{serve → engine}/session-completion-engine.ts +6 -6
- package/src/engine/telemetry.ts +109 -0
- package/src/hub/downloads.ts +116 -0
- package/src/jobs/db.ts +16 -7
- package/src/jobs/events.ts +29 -0
- package/src/jobs/executable.ts +5 -0
- package/src/jobs/host.ts +51 -0
- package/src/jobs/in-process.ts +55 -0
- package/src/{contracts/jobs.ts → jobs/protocol.ts} +7 -9
- package/src/jobs/runner.ts +150 -231
- package/src/jobs/service.ts +75 -0
- package/src/jobs/sse.ts +59 -23
- package/src/jobs/worker-memory.ts +46 -0
- package/src/jobs/worker-process.ts +163 -0
- package/src/jobs/worker-supervisor.ts +258 -0
- package/src/modules.ts +58 -0
- package/src/publishing/catalog-hub.ts +18 -0
- package/src/publishing/credentials.ts +31 -0
- package/src/publishing/upload.ts +37 -0
- package/src/residency/leased-adapters.ts +21 -0
- package/src/residency/local-records.ts +16 -0
- package/src/residency/model-residency.ts +405 -0
- package/src/residency/record-index.ts +28 -0
- package/src/residency/resident-estimate.ts +50 -0
- package/src/{anthropic.ts → server/anthropic.ts} +3 -81
- package/src/server/cache-routes.ts +36 -0
- package/src/{serve → server}/chat-request.ts +10 -10
- package/src/{serve → server}/chat-stage.ts +14 -14
- package/src/{client.ts → server/client.ts} +28 -2
- package/src/{serve → server}/completion-executor.ts +33 -41
- package/src/server/decision-body.ts +55 -0
- package/src/server/decision-routes.ts +134 -0
- package/src/server/decision-wire.ts +228 -0
- package/src/{serve → server}/discovery-routes.ts +25 -82
- package/src/{serve → server}/http.ts +25 -13
- package/src/{serve → server}/inference-request.ts +5 -5
- package/src/server/job-routes.ts +20 -0
- package/src/server/media-prompt.ts +63 -0
- package/src/server/memory-completion-client.ts +46 -0
- package/src/server/model-routes.ts +192 -0
- package/src/{serve → server}/openai-wire.ts +3 -2
- package/src/{serve → server}/pipeline.ts +1 -1
- package/src/server/prompt-builder.ts +13 -0
- package/src/server/prompt-contracts.ts +17 -0
- package/src/server/proxy-routes.ts +187 -0
- package/src/server/publishing-routes.ts +41 -0
- package/src/{serve → server}/request-plan.ts +6 -8
- package/src/{serve → server}/request-prep.ts +47 -50
- package/src/{serve → server}/responses-client.ts +30 -13
- package/src/{responses.ts → server/responses.ts} +12 -93
- package/src/server/routes.ts +205 -0
- package/src/server/start.ts +72 -0
- package/src/server/status-routes.ts +108 -0
- package/src/{serve → server}/text-completion-stage.ts +25 -4
- package/src/server/text-prompt.ts +12 -0
- package/src/{serve → server}/token-streams.ts +36 -38
- package/src/server/worker-routes.ts +311 -0
- package/src/storage/paths.ts +61 -0
- package/src/web/assets.ts +37 -0
- package/src/web/{src → browser}/api.ts +5 -23
- package/src/web/browser/chat-host.ts +17 -0
- package/src/web/browser/installed-panels.ts +6 -0
- package/src/web/browser/main.ts +55 -0
- package/src/web/browser/markdown.ts +279 -0
- package/src/web/browser/memory-panel.ts +9 -0
- package/src/web/browser/palette.ts +64 -0
- package/src/web/browser/protocol.ts +13 -0
- package/src/web/browser/shell.ts +467 -0
- package/src/web/{src → browser}/status.ts +32 -36
- package/src/web/build.ts +51 -0
- package/src/web/public/app.html +1054 -0
- package/src/web/{sw.js → public/sw.js} +4 -3
- package/src/web/{vendor → public/vendor}/hljs-theme.css +2 -2
- package/src/web/{vendor → public/vendor}/hljs.js +2 -2
- package/THIRD_PARTY_LICENSES.md +0 -51
- package/docs/README.md +0 -107
- package/docs/archive/hlg-sampling.md +0 -350
- package/docs/archive/investigations/audio-input-plan.md +0 -356
- package/docs/archive/investigations/benchmark-ledger-through-v0.4.0.md +0 -3617
- package/docs/archive/investigations/chunk-finetune-experiment.md +0 -84
- package/docs/archive/investigations/colibri-glm52-port.md +0 -1486
- package/docs/archive/investigations/colibri-oracle-pin.md +0 -1284
- package/docs/archive/investigations/curve-bisector-routing.md +0 -123
- package/docs/archive/investigations/decode-roofline-lookagain.md +0 -348
- package/docs/archive/investigations/diffusion-gemma-port.md +0 -602
- package/docs/archive/investigations/dspark-handoff.md +0 -224
- package/docs/archive/investigations/expert-offload-single-user-moe.md +0 -292
- package/docs/archive/investigations/faithful-l1-consolidation.md +0 -310
- package/docs/archive/investigations/hlg-sampling-investigation.md +0 -970
- package/docs/archive/investigations/indexshare-performance-spike.md +0 -357
- package/docs/archive/investigations/kernel-perf-review-2026-07.md +0 -171
- package/docs/archive/investigations/lab-build-journal.md +0 -157
- package/docs/archive/investigations/minicpm5-decode-megakernel.md +0 -489
- package/docs/archive/investigations/openjarvis-fit.md +0 -198
- package/docs/archive/investigations/openwiki-evaluation.md +0 -249
- package/docs/archive/investigations/optimization_plan.md +0 -346
- package/docs/archive/investigations/orpo-base-uf-experiment-and-directions.md +0 -120
- package/docs/archive/investigations/orpo-flash-cce-pin-leak.md +0 -167
- package/docs/archive/investigations/orpo-uf-testing-handoff.md +0 -126
- package/docs/archive/investigations/pareto-specialized-runtime-findings.md +0 -92
- package/docs/archive/investigations/pi-builtin-investigation.md +0 -308
- package/docs/archive/investigations/pre-colibri-stabilization.md +0 -329
- package/docs/archive/investigations/qwen38-27b-campaign-log.md +0 -1092
- package/docs/archive/investigations/segmented-backward-handoff.md +0 -148
- package/docs/archive/investigations/starter-model-port-handoff.md +0 -342
- package/docs/archive/investigations/steel-flash-cce-handoff.md +0 -489
- package/docs/archive/investigations/the-dreaming-handoff.md +0 -141
- package/docs/archive/investigations/trainer-validation-experiment.md +0 -173
- package/docs/archive/planning/chat-ui-vision-memory-first.md +0 -199
- package/docs/archive/planning/chat-ui-vision.md +0 -50
- package/docs/archive/planning/chunk-task-roadmap.md +0 -77
- package/docs/archive/planning/curve-sampler-research-plan.md +0 -331
- package/docs/archive/planning/journal.md +0 -316
- package/docs/archive/planning/memory-docs-and-dag-plan.md +0 -194
- package/docs/archive/planning/pi-integration-review.md +0 -178
- package/docs/archive/planning/release-notes-v0.0.10.md +0 -57
- package/docs/archive/planning/release-notes-v0.0.11.md +0 -239
- package/docs/archive/planning/release-notes-v0.0.12.md +0 -80
- package/docs/archive/planning/release-notes-v0.0.13.md +0 -77
- package/docs/archive/planning/release-notes-v0.0.9.md +0 -122
- package/docs/archive/planning/release-notes-v0.2.0.md +0 -49
- package/docs/archive/planning/release-notes-v0.3.0.md +0 -35
- package/docs/archive/planning/release-notes-v0.4.0.md +0 -45
- package/docs/archive/planning/release-notes-v0.5.0.md +0 -37
- package/docs/archive/planning/web-ui-pass-plan.md +0 -213
- package/docs/archive/planning/website-readme-pass-plan.md +0 -224
- package/docs/design/batching.md +0 -2054
- package/docs/design/decode-speed-program.md +0 -2404
- package/docs/design/dreaming-nightly-pipeline.md +0 -684
- package/docs/design/generic-model-support.md +0 -821
- package/docs/design/kv-cache.md +0 -993
- package/docs/design/orpo-training.md +0 -680
- package/docs/design/speculative-decoding.md +0 -2129
- package/docs/design/turboquant.md +0 -2225
- package/docs/design/unified-engine-frontier-plan.md +0 -1987
- package/docs/design/web-chat-redesign.md +0 -695
- package/docs/planning/PRODUCT_ROADMAP.md +0 -593
- package/docs/planning/ResearchTopics.md +0 -119
- package/docs/planning/public-readiness-handoff.md +0 -244
- package/docs/planning/public-readiness.md +0 -149
- package/docs/planning/video-series.md +0 -421
- package/docs/reference/benchmarks.md +0 -3587
- package/docs/reference/cli.md +0 -646
- package/docs/reference/distribution.md +0 -380
- package/docs/reference/environment.md +0 -494
- package/docs/reference/glossary.md +0 -39
- package/docs/reference/library-api.md +0 -688
- package/docs/reference/memory.md +0 -260
- package/docs/reference/models.md +0 -346
- package/docs/reference/server-api.md +0 -1945
- package/docs/reference/server-config.md +0 -973
- package/docs/reference/training.md +0 -462
- package/docs/reference/troubleshooting.md +0 -158
- package/src/admission-plan.ts +0 -18
- package/src/audio/audiotoolbox.ts +0 -112
- package/src/audio/conformer.ts +0 -655
- package/src/audio/decode.ts +0 -133
- package/src/audio/features.ts +0 -202
- package/src/audio/mic-capture.ts +0 -98
- package/src/audio/silero-vad.ts +0 -379
- package/src/audio/transcode.ts +0 -67
- package/src/audio/whisper-decode.ts +0 -774
- package/src/audio/whisper-fast.ts +0 -732
- package/src/audio/whisper-format.ts +0 -73
- package/src/audio/whisper-mel.ts +0 -185
- package/src/audio/whisper-timing.ts +0 -274
- package/src/audio/whisper-tokenizer.ts +0 -239
- package/src/audio/whisper-transcribe.ts +0 -410
- package/src/backends/mlx/assistant-target.ts +0 -46
- package/src/backends/mlx/autoregressive-draft-rows.ts +0 -120
- package/src/backends/mlx/autoregressive.ts +0 -129
- package/src/backends/mlx/batch-group.ts +0 -1874
- package/src/backends/mlx/cache-attachment.ts +0 -67
- package/src/backends/mlx/cache-layout.ts +0 -31
- package/src/backends/mlx/checkpoint-state.ts +0 -53
- package/src/backends/mlx/continuation-persistence.ts +0 -107
- package/src/backends/mlx/continuation-request.ts +0 -43
- package/src/backends/mlx/continuation.ts +0 -77
- package/src/backends/mlx/diffusion.ts +0 -39
- package/src/backends/mlx/draft-target.ts +0 -32
- package/src/backends/mlx/encoder-cache.ts +0 -25
- package/src/backends/mlx/evaluation.ts +0 -105
- package/src/backends/mlx/fill-append.ts +0 -58
- package/src/backends/mlx/fill-group.ts +0 -497
- package/src/backends/mlx/full-kv-row-append.ts +0 -32
- package/src/backends/mlx/full-kv-row-donor.ts +0 -54
- package/src/backends/mlx/full-transitioning-kv-rows.ts +0 -65
- package/src/backends/mlx/gateway-binding.ts +0 -174
- package/src/backends/mlx/graph.ts +0 -38
- package/src/backends/mlx/kv-codec.ts +0 -9
- package/src/backends/mlx/kv-maintenance.ts +0 -161
- package/src/backends/mlx/kv-tensor-rows.ts +0 -212
- package/src/backends/mlx/memory-client.ts +0 -189
- package/src/backends/mlx/methods.ts +0 -90
- package/src/backends/mlx/mixed-iteration.ts +0 -62
- package/src/backends/mlx/model-prompt.ts +0 -178
- package/src/backends/mlx/model-serving.ts +0 -80
- package/src/backends/mlx/paged-cache-codec.ts +0 -27
- package/src/backends/mlx/prefill-cohort.ts +0 -100
- package/src/backends/mlx/prefill-policy.ts +0 -26
- package/src/backends/mlx/prefill-rows.ts +0 -244
- package/src/backends/mlx/prefill.ts +0 -33
- package/src/backends/mlx/projected-context-rows.ts +0 -72
- package/src/backends/mlx/prompt-input.ts +0 -29
- package/src/backends/mlx/qwen-prompt-input.ts +0 -40
- package/src/backends/mlx/request-policy.ts +0 -20
- package/src/backends/mlx/request-state-policy.ts +0 -54
- package/src/backends/mlx/rollback.ts +0 -92
- package/src/backends/mlx/row-sampling.ts +0 -49
- package/src/backends/mlx/serial-executor.ts +0 -251
- package/src/backends/mlx/speculative-group.ts +0 -438
- package/src/backends/mlx/speculative-round.ts +0 -161
- package/src/backends/mlx/speculative.ts +0 -131
- package/src/backends/mlx/state-rows.ts +0 -104
- package/src/backends/mlx/state-views.ts +0 -46
- package/src/backends/mlx/target-layout-capability.ts +0 -35
- package/src/backends/mlx/text-engine.ts +0 -73
- package/src/backends/mlx/transitioning-kv-rows.ts +0 -173
- package/src/chat-template.ts +0 -382
- package/src/cli.ts +0 -3490
- package/src/config.ts +0 -399
- package/src/contracts/completion.ts +0 -16
- package/src/contracts/execution.ts +0 -70
- package/src/contracts/generation.ts +0 -103
- package/src/contracts/host.ts +0 -7
- package/src/contracts/object-cache.ts +0 -8
- package/src/contracts/pi-provider.ts +0 -30
- package/src/contracts/pi.ts +0 -261
- package/src/contracts/prefix-cache.ts +0 -33
- package/src/contracts/resources.ts +0 -25
- package/src/contracts/sampling.ts +0 -23
- package/src/contracts/scheduling.ts +0 -54
- package/src/contracts/tsconfig.json +0 -14
- package/src/dataset/generators.ts +0 -1087
- package/src/dataset/index.ts +0 -12
- package/src/dataset/job.ts +0 -81
- package/src/dataset/llm.ts +0 -71
- package/src/dataset/registry.ts +0 -528
- package/src/diffusion/diffusion-generate.ts +0 -507
- package/src/download.ts +0 -505
- package/src/embed.ts +0 -77
- package/src/engine/admission.ts +0 -59
- package/src/engine/cancellation.ts +0 -38
- package/src/engine/engine.ts +0 -56
- package/src/engine/execution-coordinator.ts +0 -70
- package/src/engine/execution-plan.ts +0 -64
- package/src/engine/execution-tasks.ts +0 -44
- package/src/engine/namespaced-cache.ts +0 -13
- package/src/engine/resources.ts +0 -53
- package/src/engine/scheduler.ts +0 -92
- package/src/engine/session.ts +0 -247
- package/src/eval/README.md +0 -130
- package/src/eval/ifeval.ts +0 -81
- package/src/eval/kl-prompts.ts +0 -29
- package/src/eval/kl.ts +0 -590
- package/src/eval/perplexity.ts +0 -164
- package/src/eval/quality-db.ts +0 -92
- package/src/eval/reference-logits.ts +0 -207
- package/src/eval/runner.ts +0 -124
- package/src/eval/score.ts +0 -49
- package/src/eval/tasks/bfcl.ts +0 -535
- package/src/eval/tasks/gsm8k.ts +0 -98
- package/src/eval/tasks/hashhop.ts +0 -139
- package/src/eval/tasks/humaneval.ts +0 -275
- package/src/eval/tasks/ifeval.ts +0 -525
- package/src/eval/tasks/mmlu.ts +0 -157
- package/src/eval/ultrafeedback.ts +0 -60
- package/src/evaldb.ts +0 -199
- package/src/expert-io.ts +0 -393
- package/src/expert-offload-build.ts +0 -77
- package/src/expert-offload.ts +0 -76
- package/src/expert-residency.ts +0 -1134
- package/src/expert-usage.ts +0 -415
- package/src/fill/echo-index.ts +0 -188
- package/src/fill/fill-session.ts +0 -463
- package/src/fill/proposal.ts +0 -68
- package/src/fill/schema-rows.ts +0 -506
- package/src/fill/tool-boundary.ts +0 -103
- package/src/fit.ts +0 -249
- package/src/generate.ts +0 -1352
- package/src/grammar.ts +0 -604
- package/src/harness-pi.ts +0 -184
- package/src/hf-push.ts +0 -628
- package/src/hub-rest.ts +0 -242
- package/src/index.ts +0 -63
- package/src/inference/denoising.ts +0 -17
- package/src/inference/draft-acceptance.ts +0 -41
- package/src/inference/draft-output.ts +0 -49
- package/src/inference/graph.ts +0 -33
- package/src/inference/prefill.ts +0 -45
- package/src/inference/rollback.ts +0 -7
- package/src/jobs/index.ts +0 -31
- package/src/jobs/job-entry.ts +0 -107
- package/src/jobs/task-client.ts +0 -19
- package/src/jobs/types.ts +0 -2
- package/src/kv-scheme.ts +0 -279
- package/src/kv-store.ts +0 -1135
- package/src/lab/README.md +0 -22
- package/src/lab/curve/curve-designer.html +0 -303
- package/src/lab/curve/curve-sampler.ts +0 -122
- package/src/lab/expert-trace/expert-trace.ts +0 -69
- package/src/lab/paged-kv/paged-attention.ts +0 -117
- package/src/lab/paged-kv/paged-kv-rows.ts +0 -111
- package/src/lab/paged-kv/paged-kv.ts +0 -336
- package/src/library.ts +0 -40
- package/src/lora.ts +0 -408
- package/src/md-modules.d.ts +0 -54
- package/src/media-fetch.ts +0 -442
- package/src/memory/article.ts +0 -558
- package/src/memory/chunk-validation.ts +0 -150
- package/src/memory/chunk.ts +0 -318
- package/src/memory/cluster.ts +0 -351
- package/src/memory/coverage.ts +0 -134
- package/src/memory/crosslink.ts +0 -0
- package/src/memory/db.ts +0 -253
- package/src/memory/entity.ts +0 -203
- package/src/memory/gate.ts +0 -187
- package/src/memory/history.ts +0 -152
- package/src/memory/ingest.ts +0 -335
- package/src/memory/model.ts +0 -183
- package/src/memory/normalize.ts +0 -671
- package/src/memory/parse.ts +0 -22
- package/src/memory/pipeline.ts +0 -275
- package/src/memory/prompts.ts +0 -65
- package/src/memory/query.ts +0 -334
- package/src/memory/reconcile.ts +0 -734
- package/src/memory/redirect.ts +0 -108
- package/src/memory/reindex.ts +0 -325
- package/src/memory/resolve.ts +0 -307
- package/src/memory/rest.ts +0 -245
- package/src/memory/route.ts +0 -272
- package/src/memory/schedule.ts +0 -165
- package/src/memory/stages.ts +0 -777
- package/src/memory/synthesize.ts +0 -1638
- package/src/memory/tools.ts +0 -749
- package/src/memory/vault.ts +0 -1194
- package/src/memory/wikify.ts +0 -1064
- package/src/mlx/array.ts +0 -425
- package/src/mlx/autograd.ts +0 -304
- package/src/mlx/checkpoint.ts +0 -149
- package/src/mlx/compile.ts +0 -168
- package/src/mlx/custom-vjp.ts +0 -188
- package/src/mlx/ffi.ts +0 -531
- package/src/mlx/materialize.ts +0 -23
- package/src/mlx/metal-kernel.ts +0 -159
- package/src/mlx/normalized-argmax.ts +0 -116
- package/src/mlx/ops.ts +0 -1108
- package/src/mlx/quantized-rows.ts +0 -36
- package/src/mlx/token-bitmask.ts +0 -52
- package/src/mlx/turboquant-kv-decode.ts +0 -82
- package/src/mlx/turboquant-kv-inverse.ts +0 -122
- package/src/mlx/turboquant-ops.ts +0 -861
- package/src/mlx/turboquant-tables.ts +0 -237
- package/src/mmap.ts +0 -90
- package/src/model/artifact-identity.ts +0 -73
- package/src/model/batched-kv.ts +0 -235
- package/src/model/batched-mask.ts +0 -279
- package/src/model/batched-quant.ts +0 -212
- package/src/model/batched-quantized-kv.ts +0 -144
- package/src/model/batched-rotating-quant.ts +0 -353
- package/src/model/batched-rotating-state.ts +0 -149
- package/src/model/batched-rotating.ts +0 -338
- package/src/model/batched-row-storage.ts +0 -179
- package/src/model/batched-ssm.ts +0 -52
- package/src/model/batched-turboquant-kv.ts +0 -94
- package/src/model/compiled-decode.ts +0 -712
- package/src/model/decoded-kv-donor.ts +0 -12
- package/src/model/delayed-quantized-kv.ts +0 -72
- package/src/model/delayed-rotating-quantized-kv.ts +0 -93
- package/src/model/delayed-turboquant-kv.ts +0 -57
- package/src/model/diffusion-gemma.ts +0 -1026
- package/src/model/factory.ts +0 -215
- package/src/model/fingerprint.ts +0 -45
- package/src/model/flash-attention.ts +0 -694
- package/src/model/full-prefill-padding.ts +0 -56
- package/src/model/full-prefill-row.ts +0 -105
- package/src/model/gemma4-base.ts +0 -2370
- package/src/model/gemma4.ts +0 -1395
- package/src/model/generated/gemma4-12b.ts +0 -429
- package/src/model/generated/gemma4-26b.ts +0 -448
- package/src/model/generated/gemma4-e4b.ts +0 -717
- package/src/model/generated/index.ts +0 -18
- package/src/model/glm52-atlas.ts +0 -511
- package/src/model/glm52-cache.ts +0 -792
- package/src/model/glm52-config.ts +0 -262
- package/src/model/glm52-container.ts +0 -305
- package/src/model/glm52-coupling.ts +0 -388
- package/src/model/glm52-dsa.ts +0 -606
- package/src/model/glm52-expert-layout.ts +0 -253
- package/src/model/glm52-layout.ts +0 -152
- package/src/model/glm52-memory.ts +0 -500
- package/src/model/glm52-mla.ts +0 -792
- package/src/model/glm52-moe.ts +0 -223
- package/src/model/glm52-pilot.ts +0 -537
- package/src/model/glm52-reference.ts +0 -445
- package/src/model/glm52-residency.ts +0 -557
- package/src/model/glm52-resident-weights.ts +0 -135
- package/src/model/glm52-streamed-experts.ts +0 -453
- package/src/model/glm52-streamed-metal.ts +0 -610
- package/src/model/glm52-weights.ts +0 -503
- package/src/model/glm52.ts +0 -1132
- package/src/model/implementation.ts +0 -69
- package/src/model/kv-attention-view.ts +0 -70
- package/src/model/minicpm5.ts +0 -376
- package/src/model/profile.ts +0 -468
- package/src/model/quantized-append-attention.ts +0 -31
- package/src/model/qwen-append.ts +0 -12
- package/src/model/qwen-conv.ts +0 -52
- package/src/model/qwen3-delta-state.ts +0 -74
- package/src/model/qwen3-delta.ts +0 -451
- package/src/model/qwen3-moe.ts +0 -404
- package/src/model/qwen3-mrope.ts +0 -250
- package/src/model/qwen3.ts +0 -277
- package/src/model/qwen3_5-checkpoint.ts +0 -164
- package/src/model/qwen3_5.ts +0 -881
- package/src/model/rotating-kv-donor.ts +0 -35
- package/src/model/rotating-kv-layout.ts +0 -262
- package/src/model/rotating-row-transaction.ts +0 -35
- package/src/model/speculative-rotating-kv.ts +0 -101
- package/src/model/ssm-prefill-padding.ts +0 -39
- package/src/model/support.ts +0 -110
- package/src/model/token-groups.ts +0 -66
- package/src/model/trellis-balanced-scatter.ts +0 -51
- package/src/model/trellis-linear.ts +0 -672
- package/src/model/trellis-shared-m.ts +0 -151
- package/src/model/trellis-shared-scatter.ts +0 -144
- package/src/model/trellis-splitk-prefill.ts +0 -158
- package/src/model/trellis-tiled-prefill.ts +0 -136
- package/src/model/trellis-vector-expand.ts +0 -62
- package/src/model/trellis-wide-prefill.ts +0 -94
- package/src/model/turboquant-codec.ts +0 -89
- package/src/model/universal/archs.ts +0 -332
- package/src/model/universal/dense.ts +0 -612
- package/src/model/universal/modules.ts +0 -170
- package/src/model/universal/rope.ts +0 -381
- package/src/model/whisper.ts +0 -435
- package/src/native/expert_io.c +0 -523
- package/src/native/frame_extract.swift +0 -140
- package/src/native/mic_capture.swift +0 -92
- package/src/native-pack.ts +0 -163
- package/src/pi-provider.ts +0 -53
- package/src/pi-session.ts +0 -81
- package/src/pi-terminal.ts +0 -274
- package/src/pi-web.ts +0 -1940
- package/src/preflight.ts +0 -158
- package/src/prompt-cache.ts +0 -682
- package/src/quantize/allocator.ts +0 -627
- package/src/quantize/atomic-output.ts +0 -38
- package/src/quantize/calibration.ts +0 -181
- package/src/quantize/config-writer.ts +0 -145
- package/src/quantize/index.ts +0 -84
- package/src/quantize/job.ts +0 -162
- package/src/quantize/quantizer.ts +0 -666
- package/src/quantize/rotate.ts +0 -603
- package/src/quantize/safetensors-writer.ts +0 -249
- package/src/quantize/sensitivity.ts +0 -216
- package/src/quantize/trellis.ts +0 -586
- package/src/quantize/weight-transform.ts +0 -292
- package/src/registry.ts +0 -602
- package/src/runtime-config.ts +0 -77
- package/src/safetensors.ts +0 -165
- package/src/sampler.ts +0 -1012
- package/src/serve/admin-routes.ts +0 -184
- package/src/serve/assets/curve-terrain.html +0 -105
- package/src/serve/assets/training-inference-map.html +0 -575
- package/src/serve/audio-routes.ts +0 -323
- package/src/serve/aux-routes.ts +0 -132
- package/src/serve/batch-scheduler.ts +0 -8
- package/src/serve/checkpoint-identity.ts +0 -64
- package/src/serve/generation-gateway.ts +0 -537
- package/src/serve/isolate.ts +0 -609
- package/src/serve/kv-budget.ts +0 -14
- package/src/serve/lab-routes.ts +0 -341
- package/src/serve/lane-registry.ts +0 -45
- package/src/serve/model-admin-routes.ts +0 -155
- package/src/serve/model-binding.ts +0 -49
- package/src/serve/model-host.ts +0 -511
- package/src/serve/parent-application.ts +0 -56
- package/src/serve/prompt-response-trace.ts +0 -161
- package/src/serve/response-lease.ts +0 -24
- package/src/serve/session-search.ts +0 -326
- package/src/serve/static-routes.ts +0 -101
- package/src/serve/transcription-server.ts +0 -64
- package/src/serve/transcription-service.ts +0 -502
- package/src/server.ts +0 -1282
- package/src/spec/assistant-rows.ts +0 -93
- package/src/spec/assistant-source.ts +0 -126
- package/src/spec/cache-identity.ts +0 -22
- package/src/spec/deepspec-rows.ts +0 -29
- package/src/spec/deepspec-source.ts +0 -145
- package/src/spec/dflash-source.ts +0 -149
- package/src/spec/drafter.ts +0 -304
- package/src/spec/dspark/ab-stats.ts +0 -136
- package/src/spec/dspark/calibration.ts +0 -112
- package/src/spec/dspark/data-dflash.ts +0 -156
- package/src/spec/dspark/data.ts +0 -201
- package/src/spec/dspark/deepspec-module.ts +0 -776
- package/src/spec/dspark/generate-dflash.ts +0 -245
- package/src/spec/dspark/generate.ts +0 -290
- package/src/spec/dspark/loader.ts +0 -23
- package/src/spec/dspark/loss.ts +0 -213
- package/src/spec/dspark/module-dflash.ts +0 -560
- package/src/spec/dspark/module.ts +0 -540
- package/src/spec/dspark/quantize-drafter.ts +0 -91
- package/src/spec/dspark/sample.ts +0 -112
- package/src/spec/generate.ts +0 -276
- package/src/spec/glm52-mtp-graph.ts +0 -29
- package/src/spec/glm52-mtp-rows.ts +0 -132
- package/src/spec/glm52-mtp-source.ts +0 -318
- package/src/spec/glm52-mtp-state.ts +0 -33
- package/src/spec/ngram-source.ts +0 -226
- package/src/spec/projected-draft-rows.ts +0 -73
- package/src/spec/qwen-mtp-module.ts +0 -171
- package/src/spec/qwen-mtp-rows.ts +0 -230
- package/src/spec/qwen-mtp-source.ts +0 -386
- package/src/spec/qwen-mtp-state.ts +0 -30
- package/src/spec/serve-loop.ts +0 -587
- package/src/spec/source.ts +0 -228
- package/src/spec/two-model-rows.ts +0 -62
- package/src/spec/two-model.ts +0 -113
- package/src/ssd-cache.ts +0 -431
- package/src/ssd-durability.ts +0 -209
- package/src/storage/host-buffer.ts +0 -24
- package/src/storage/kv-writer.ts +0 -83
- package/src/storage/kv-writer.worker.js +0 -264
- package/src/storage/retention-policy.ts +0 -37
- package/src/tiered-prompt-cache.ts +0 -45
- package/src/tokenizer.ts +0 -335
- package/src/tool-approvals.ts +0 -131
- package/src/tool-call.ts +0 -695
- package/src/train/dataset.ts +0 -484
- package/src/train/diffusion-lora.ts +0 -216
- package/src/train/export.ts +0 -32
- package/src/train/flash-cce.ts +0 -1234
- package/src/train/forward.ts +0 -266
- package/src/train/fuse.ts +0 -243
- package/src/train/index.ts +0 -46
- package/src/train/job.ts +0 -205
- package/src/train/lora-params.ts +0 -254
- package/src/train/loss.ts +0 -1591
- package/src/train/merge.ts +0 -195
- package/src/train/optimizer.ts +0 -183
- package/src/train/prefix-shared.ts +0 -490
- package/src/train/rank.ts +0 -137
- package/src/train/segmented.ts +0 -1828
- package/src/train/steel-qmm-header.ts +0 -1461
- package/src/train/trainer.ts +0 -1221
- package/src/train/watch.ts +0 -542
- package/src/vision/diffusion-vision.ts +0 -620
- package/src/vision/embedder.ts +0 -176
- package/src/vision/preprocess.ts +0 -247
- package/src/vision/prompt.ts +0 -409
- package/src/vision/qwen3vl-preprocess.ts +0 -409
- package/src/vision/qwen3vl-prompt.ts +0 -233
- package/src/vision/qwen3vl-tower.ts +0 -566
- package/src/vision/siglip.ts +0 -607
- package/src/vision/video-frames.ts +0 -148
- package/src/web/app.html +0 -2155
- package/src/web/app.js +0 -5863
- package/src/web/skills/memory/SKILL.md +0 -120
- package/src/web/skills/web-research/SKILL.md +0 -46
- package/src/web/skills.ts +0 -97
- package/src/web/src/adapters-panel.ts +0 -288
- package/src/web/src/assistant.ts +0 -451
- package/src/web/src/chat.ts +0 -1060
- package/src/web/src/composer.ts +0 -962
- package/src/web/src/dataset.ts +0 -132
- package/src/web/src/finetune.ts +0 -267
- package/src/web/src/hub.ts +0 -428
- package/src/web/src/main.ts +0 -98
- package/src/web/src/markdown.ts +0 -0
- package/src/web/src/memory-panel.ts +0 -572
- package/src/web/src/model-picker.ts +0 -159
- package/src/web/src/palette.ts +0 -346
- package/src/web/src/protocol.ts +0 -26
- package/src/web/src/quantize.ts +0 -167
- package/src/web/src/rag.ts +0 -274
- package/src/web/src/sessions.ts +0 -469
- package/src/web/src/shell.ts +0 -804
- package/src/web/src/ui-catalog.ts +0 -78
- package/src/web/src/voice.ts +0 -141
- package/src/web/vendor/README.md +0 -61
- package/src/web-assets.ts +0 -47
- package/src/web-tools.ts +0 -478
- package/src/weights.ts +0 -255
- /package/src/{serve → server}/completion-sink.ts +0 -0
- /package/src/{serve → server}/generated-token-history.ts +0 -0
- /package/src/web/{icon.svg → public/icon.svg} +0 -0
- /package/src/web/{manifest.webmanifest → public/manifest.webmanifest} +0 -0
- /package/src/web/{vendor → public/vendor}/hljs-LICENSE +0 -0
package/src/memory/vault.ts
DELETED
|
@@ -1,1194 +0,0 @@
|
|
|
1
|
-
// mlx-bun memory — the wiki vault on disk.
|
|
2
|
-
//
|
|
3
|
-
// "Memory" is the user-facing capability; the store is a plain Markdown wiki at
|
|
4
|
-
// ~/.mlx-bun/wiki/ — articles the local assistant reads to remember you across
|
|
5
|
-
// sessions. One self-contained, portable folder: copy it and your memory moves
|
|
6
|
-
// with you. It's a valid Obsidian vault for free.
|
|
7
|
-
//
|
|
8
|
-
// The article read/search/link helpers are ported from lucien (~/Code/lucien),
|
|
9
|
-
// an already-working system at real scale — they're pure filesystem functions
|
|
10
|
-
// with no MCP coupling, so they drop straight in. We keep lucien's good
|
|
11
|
-
// descriptive concepts (articles, [[wikilinks]], Meta governance pages,
|
|
12
|
-
// conv: citations) and drop the mythology. See docs/design/dreaming-nightly-pipeline.md.
|
|
13
|
-
//
|
|
14
|
-
// M0 is the read path: setup the vault, then list / read / search / index it.
|
|
15
|
-
// Synthesis (writing articles from conversations) is M1 and lives elsewhere.
|
|
16
|
-
|
|
17
|
-
import { access, lstat, mkdir, readdir, readFile, readlink, stat, symlink, unlink, writeFile } from "node:fs/promises";
|
|
18
|
-
import { homedir } from "node:os";
|
|
19
|
-
import { isAbsolute, join } from "node:path";
|
|
20
|
-
import { spawn } from "node:child_process";
|
|
21
|
-
import { isFenceLine } from "./article";
|
|
22
|
-
import { runtimeValue } from "../runtime-config";
|
|
23
|
-
|
|
24
|
-
// ---- paths -----------------------------------------------------------
|
|
25
|
-
|
|
26
|
-
/** Root of the memory vault: ~/.mlx-bun/wiki (override with MLX_BUN_WIKI). */
|
|
27
|
-
export function vaultRoot(): string {
|
|
28
|
-
const override = runtimeValue("MLX_BUN_WIKI")?.trim();
|
|
29
|
-
if (override) return override.replace(/^~(?=$|[/\\])/, homedir());
|
|
30
|
-
return join(homedir(), ".mlx-bun", "wiki");
|
|
31
|
-
}
|
|
32
|
-
|
|
33
|
-
export function articlesDir(root = vaultRoot()): string {
|
|
34
|
-
return join(root, "articles");
|
|
35
|
-
}
|
|
36
|
-
|
|
37
|
-
export function referenceDir(root = vaultRoot()): string {
|
|
38
|
-
return join(root, "Reference");
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
// ---- types -----------------------------------------------------------
|
|
42
|
-
|
|
43
|
-
export interface TocEntry {
|
|
44
|
-
depth: number;
|
|
45
|
-
title: string;
|
|
46
|
-
anchor: string;
|
|
47
|
-
}
|
|
48
|
-
|
|
49
|
-
export interface SearchHit {
|
|
50
|
-
article: string;
|
|
51
|
-
anchor: string | null;
|
|
52
|
-
line: number;
|
|
53
|
-
excerpt: string;
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
export interface ArticleSearchSummary {
|
|
57
|
-
article: string;
|
|
58
|
-
occurrences: number;
|
|
59
|
-
/** Distinct query terms that matched this article (multi-term queries). */
|
|
60
|
-
matched_terms?: string[];
|
|
61
|
-
/** Number of query terms that appear in the article's title/stem. */
|
|
62
|
-
title_matches?: number;
|
|
63
|
-
}
|
|
64
|
-
|
|
65
|
-
// ---- markdown helpers (ported from lucien) ---------------------------
|
|
66
|
-
|
|
67
|
-
export function slugifyHeading(title: string): string {
|
|
68
|
-
return title
|
|
69
|
-
.trim()
|
|
70
|
-
.toLowerCase()
|
|
71
|
-
.replace(/[^a-z0-9]+/g, "-")
|
|
72
|
-
.replace(/^-+|-+$/g, "");
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
/** Non-overlapping substring matches (whole file). */
|
|
76
|
-
export function countSubstringOccurrences(text: string, needle: string, caseSensitive: boolean): number {
|
|
77
|
-
const n = needle.trim();
|
|
78
|
-
if (!n) return 0;
|
|
79
|
-
const haystack = caseSensitive ? text : text.toLowerCase();
|
|
80
|
-
const nd = caseSensitive ? n : n.toLowerCase();
|
|
81
|
-
let count = 0;
|
|
82
|
-
let pos = 0;
|
|
83
|
-
for (;;) {
|
|
84
|
-
const idx = haystack.indexOf(nd, pos);
|
|
85
|
-
if (idx === -1) break;
|
|
86
|
-
count++;
|
|
87
|
-
pos = idx + nd.length;
|
|
88
|
-
}
|
|
89
|
-
return count;
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
/** Heading outside fences; depth 1 = `#`, … 6 = `######`. */
|
|
93
|
-
function parseHeadingLine(line: string): { depth: number; title: string } | null {
|
|
94
|
-
const m = /^(#{1,6})\s+(.+)$/.exec(line);
|
|
95
|
-
if (!m) return null;
|
|
96
|
-
return { depth: m[1]!.length, title: m[2]!.trim() };
|
|
97
|
-
}
|
|
98
|
-
|
|
99
|
-
export function parseToc(markdown: string): TocEntry[] {
|
|
100
|
-
const lines = markdown.split(/\r?\n/);
|
|
101
|
-
let inFence = false;
|
|
102
|
-
const toc: TocEntry[] = [];
|
|
103
|
-
for (const line of lines) {
|
|
104
|
-
if (isFenceLine(line)) {
|
|
105
|
-
inFence = !inFence;
|
|
106
|
-
continue;
|
|
107
|
-
}
|
|
108
|
-
if (inFence) continue;
|
|
109
|
-
const h = parseHeadingLine(line);
|
|
110
|
-
if (h) toc.push({ depth: h.depth, title: h.title, anchor: slugifyHeading(h.title) });
|
|
111
|
-
}
|
|
112
|
-
return toc;
|
|
113
|
-
}
|
|
114
|
-
|
|
115
|
-
function fenceAfterLine(lines: string[], lineIdx: number): boolean {
|
|
116
|
-
let inFence = false;
|
|
117
|
-
for (let i = 0; i <= lineIdx; i++) {
|
|
118
|
-
if (isFenceLine(lines[i]!)) inFence = !inFence;
|
|
119
|
-
}
|
|
120
|
-
return inFence;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
/**
|
|
124
|
-
* First heading with matching anchor wins. Includes the heading line. Stops
|
|
125
|
-
* before the next heading (outside fences) of equal or shallower depth.
|
|
126
|
-
*/
|
|
127
|
-
export function extractSection(markdown: string, anchor: string): string | null {
|
|
128
|
-
const lines = markdown.split(/\r?\n/);
|
|
129
|
-
let inFence = false;
|
|
130
|
-
let startIdx = -1;
|
|
131
|
-
let startDepth = 0;
|
|
132
|
-
for (let i = 0; i < lines.length; i++) {
|
|
133
|
-
const line = lines[i]!;
|
|
134
|
-
if (isFenceLine(line)) {
|
|
135
|
-
inFence = !inFence;
|
|
136
|
-
continue;
|
|
137
|
-
}
|
|
138
|
-
if (inFence) continue;
|
|
139
|
-
const h = parseHeadingLine(line);
|
|
140
|
-
if (!h) continue;
|
|
141
|
-
if (slugifyHeading(h.title) === anchor) {
|
|
142
|
-
startIdx = i;
|
|
143
|
-
startDepth = h.depth;
|
|
144
|
-
break;
|
|
145
|
-
}
|
|
146
|
-
}
|
|
147
|
-
if (startIdx < 0) return null;
|
|
148
|
-
|
|
149
|
-
let fence = fenceAfterLine(lines, startIdx);
|
|
150
|
-
let endIdxExclusive = lines.length;
|
|
151
|
-
for (let j = startIdx + 1; j < lines.length; j++) {
|
|
152
|
-
const line = lines[j]!;
|
|
153
|
-
if (isFenceLine(line)) {
|
|
154
|
-
fence = !fence;
|
|
155
|
-
continue;
|
|
156
|
-
}
|
|
157
|
-
if (fence) continue;
|
|
158
|
-
const h = parseHeadingLine(line);
|
|
159
|
-
if (h && h.depth <= startDepth) {
|
|
160
|
-
endIdxExclusive = j;
|
|
161
|
-
break;
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
return lines.slice(startIdx, endIdxExclusive).join("\n");
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
// ---- article/reference addressing ------------------------------------
|
|
168
|
-
|
|
169
|
-
function normalizeBareStem(article: string): string {
|
|
170
|
-
const t = article.trim();
|
|
171
|
-
if (!t) throw new Error("article name is required");
|
|
172
|
-
const stem = t.endsWith(".md") ? t.slice(0, -3).trim() : t;
|
|
173
|
-
if (!stem) throw new Error("article name is invalid");
|
|
174
|
-
if (stem.includes("..")) throw new Error("article must not contain ..");
|
|
175
|
-
if (isAbsolute(stem)) throw new Error("article must be a bare filename, not a path");
|
|
176
|
-
if (stem.includes("/") || stem.includes("\\")) throw new Error("article must not contain path separators");
|
|
177
|
-
if (stem.startsWith(".")) throw new Error("article must not be hidden or relative");
|
|
178
|
-
return stem;
|
|
179
|
-
}
|
|
180
|
-
|
|
181
|
-
function normalizeMemoryDocId(article: string): { kind: "article" | "reference"; stem: string; id: string } {
|
|
182
|
-
const raw = article.trim();
|
|
183
|
-
if (/^Reference[/\\]/.test(raw)) {
|
|
184
|
-
const rest = raw.replace(/^Reference[/\\]/, "");
|
|
185
|
-
const stem = normalizeBareStem(rest);
|
|
186
|
-
return { kind: "reference", stem, id: `Reference/${stem}` };
|
|
187
|
-
}
|
|
188
|
-
const stem = normalizeBareStem(raw);
|
|
189
|
-
return { kind: "article", stem, id: stem };
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
function normalizeArticleStem(article: string): string {
|
|
193
|
-
const doc = normalizeMemoryDocId(article);
|
|
194
|
-
if (doc.kind !== "article") throw new Error("expected a user article, not a Reference document");
|
|
195
|
-
return doc.stem;
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
async function pathExists(path: string): Promise<boolean> {
|
|
199
|
-
try {
|
|
200
|
-
await access(path);
|
|
201
|
-
return true;
|
|
202
|
-
} catch {
|
|
203
|
-
return false;
|
|
204
|
-
}
|
|
205
|
-
}
|
|
206
|
-
|
|
207
|
-
export async function resolveArticlePath(root: string, article: string): Promise<string> {
|
|
208
|
-
const doc = normalizeMemoryDocId(article);
|
|
209
|
-
const base = doc.kind === "reference" ? referenceDir(root) : articlesDir(root);
|
|
210
|
-
const filePath = join(base, `${doc.stem}.md`);
|
|
211
|
-
if (!(await pathExists(filePath))) {
|
|
212
|
-
throw new Error(`${doc.kind === "reference" ? "Reference" : "Article"} not found: ${doc.id}.md under ${base}`);
|
|
213
|
-
}
|
|
214
|
-
return filePath;
|
|
215
|
-
}
|
|
216
|
-
|
|
217
|
-
export async function readArticle(root: string, article: string): Promise<{ path: string; content: string }> {
|
|
218
|
-
const path = await resolveArticlePath(root, article);
|
|
219
|
-
return { path, content: await readFile(path, "utf8") };
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
/** Vault-relative path (`articles/<stem>.md` or `Reference/<stem>.md`) for an
|
|
223
|
-
* existing article/reference doc — the safe argv piece for git plumbing
|
|
224
|
-
* (articleHistory/articleDiff), resolved via the same name normalization/
|
|
225
|
-
* existence check as readArticle so a caller can't smuggle a path-traversal
|
|
226
|
-
* or shell-metacharacter name through. Throws if the doc doesn't exist. */
|
|
227
|
-
export async function resolveArticleRelPath(root: string, article: string): Promise<string> {
|
|
228
|
-
const doc = normalizeMemoryDocId(article);
|
|
229
|
-
await resolveArticlePath(root, article); // existence check, reuses the same normalization
|
|
230
|
-
return doc.kind === "reference" ? join("Reference", `${doc.stem}.md`) : join("articles", `${doc.stem}.md`);
|
|
231
|
-
}
|
|
232
|
-
|
|
233
|
-
/** Filename stems for every `*.md` under articles/, sorted alphabetically. */
|
|
234
|
-
export async function listArticles(root: string): Promise<string[]> {
|
|
235
|
-
let names: string[];
|
|
236
|
-
try {
|
|
237
|
-
names = await readdir(articlesDir(root));
|
|
238
|
-
} catch {
|
|
239
|
-
return [];
|
|
240
|
-
}
|
|
241
|
-
return names
|
|
242
|
-
.filter((n) => n.endsWith(".md") && !n.startsWith("."))
|
|
243
|
-
.map((n) => n.slice(0, -3))
|
|
244
|
-
.sort((a, b) => a.localeCompare(b));
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
/** Reference document ids, prefixed so tools can pass them back to memory_read. */
|
|
248
|
-
export async function listReferenceDocs(root: string): Promise<string[]> {
|
|
249
|
-
let names: string[];
|
|
250
|
-
try {
|
|
251
|
-
names = await readdir(referenceDir(root));
|
|
252
|
-
} catch {
|
|
253
|
-
return [];
|
|
254
|
-
}
|
|
255
|
-
return names
|
|
256
|
-
.filter((n) => n.endsWith(".md") && !n.startsWith("."))
|
|
257
|
-
.map((n) => `Reference/${n.slice(0, -3)}`)
|
|
258
|
-
.sort((a, b) => a.localeCompare(b));
|
|
259
|
-
}
|
|
260
|
-
|
|
261
|
-
export async function listMemoryDocuments(root: string): Promise<string[]> {
|
|
262
|
-
return [...(await listArticles(root)), ...(await listReferenceDocs(root))];
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
// ---- wikilinks --------------------------------------------------------
|
|
266
|
-
|
|
267
|
-
const wikilinkInnerRe = /\[\[([^\]]+)\]\]/g;
|
|
268
|
-
|
|
269
|
-
/** Raw wikilink targets from prose only (not fenced code); includes conv: etc. */
|
|
270
|
-
export function extractWikilinkTargets(markdown: string): string[] {
|
|
271
|
-
const lines = markdown.split(/\r?\n/);
|
|
272
|
-
let inFence = false;
|
|
273
|
-
const out: string[] = [];
|
|
274
|
-
for (const line of lines) {
|
|
275
|
-
if (isFenceLine(line)) {
|
|
276
|
-
inFence = !inFence;
|
|
277
|
-
continue;
|
|
278
|
-
}
|
|
279
|
-
if (inFence) continue;
|
|
280
|
-
wikilinkInnerRe.lastIndex = 0;
|
|
281
|
-
let m: RegExpExecArray | null;
|
|
282
|
-
while ((m = wikilinkInnerRe.exec(line)) !== null) {
|
|
283
|
-
const inner = m[1]!.trim();
|
|
284
|
-
if (!inner || /^conv:/i.test(inner)) continue;
|
|
285
|
-
const hash = inner.split("|")[0]!.split("#")[0]!.trim();
|
|
286
|
-
if (hash) out.push(hash);
|
|
287
|
-
}
|
|
288
|
-
}
|
|
289
|
-
return out;
|
|
290
|
-
}
|
|
291
|
-
|
|
292
|
-
/** Map wikilink display text to an existing article stem (underscore filenames). */
|
|
293
|
-
export function resolveWikilinkToStem(raw: string, stems: Set<string>): string | null {
|
|
294
|
-
const base = raw.trim();
|
|
295
|
-
if (!base) return null;
|
|
296
|
-
if (stems.has(base)) return base;
|
|
297
|
-
const underscored = base.replace(/\s+/g, "_");
|
|
298
|
-
if (stems.has(underscored)) return underscored;
|
|
299
|
-
const lower = underscored.toLowerCase();
|
|
300
|
-
const spacedLower = base.replace(/\s+/g, " ").trim().toLowerCase();
|
|
301
|
-
for (const stem of stems) {
|
|
302
|
-
if (stem.toLowerCase() === lower) return stem;
|
|
303
|
-
if (stem.replace(/_/g, " ").trim().toLowerCase() === spacedLower) return stem;
|
|
304
|
-
}
|
|
305
|
-
return null;
|
|
306
|
-
}
|
|
307
|
-
|
|
308
|
-
export async function getArticleLinks(root: string, article: string): Promise<{ outbound: string[]; inbound: string[] }> {
|
|
309
|
-
const stem = normalizeArticleStem(article);
|
|
310
|
-
const stems = await listArticles(root);
|
|
311
|
-
const stemSet = new Set(stems);
|
|
312
|
-
await resolveArticlePath(root, stem);
|
|
313
|
-
|
|
314
|
-
const dir = articlesDir(root);
|
|
315
|
-
const outboundByStem = new Map<string, Set<string>>();
|
|
316
|
-
for (const name of stems) {
|
|
317
|
-
const content = await readFile(join(dir, `${name}.md`), "utf8");
|
|
318
|
-
const resolved = new Set<string>();
|
|
319
|
-
for (const t of extractWikilinkTargets(content)) {
|
|
320
|
-
const r = resolveWikilinkToStem(t, stemSet);
|
|
321
|
-
if (r !== null && r !== name) resolved.add(r);
|
|
322
|
-
}
|
|
323
|
-
outboundByStem.set(name, resolved);
|
|
324
|
-
}
|
|
325
|
-
|
|
326
|
-
const outbound = [...(outboundByStem.get(stem) ?? new Set<string>())].sort((a, b) => a.localeCompare(b));
|
|
327
|
-
const inbound: string[] = [];
|
|
328
|
-
for (const [from, targets] of outboundByStem) {
|
|
329
|
-
if (from !== stem && targets.has(stem)) inbound.push(from);
|
|
330
|
-
}
|
|
331
|
-
inbound.sort((a, b) => a.localeCompare(b));
|
|
332
|
-
return { outbound, inbound };
|
|
333
|
-
}
|
|
334
|
-
|
|
335
|
-
// ---- search -----------------------------------------------------------
|
|
336
|
-
|
|
337
|
-
const STOPWORDS = new Set([
|
|
338
|
-
"the", "a", "an", "of", "for", "to", "in", "on", "at", "and", "or", "is", "it", "as", "with",
|
|
339
|
-
]);
|
|
340
|
-
|
|
341
|
-
export async function searchArticles(
|
|
342
|
-
root: string,
|
|
343
|
-
query: string,
|
|
344
|
-
options?: { limit?: number; case_sensitive?: boolean; scope?: "all" | "articles" | "reference" },
|
|
345
|
-
): Promise<{ summaries: ArticleSearchSummary[]; hits: SearchHit[] }> {
|
|
346
|
-
const limit = options?.limit ?? 50;
|
|
347
|
-
const caseSensitive = options?.case_sensitive ?? false;
|
|
348
|
-
const q = query.trim();
|
|
349
|
-
if (!q) throw new Error("search query is required");
|
|
350
|
-
|
|
351
|
-
const scope = options?.scope ?? "all";
|
|
352
|
-
const docs: { id: string; stem: string; path: string }[] = [];
|
|
353
|
-
if (scope === "all" || scope === "articles") {
|
|
354
|
-
for (const stem of await listArticles(root)) {
|
|
355
|
-
docs.push({ id: stem, stem, path: join(articlesDir(root), `${stem}.md`) });
|
|
356
|
-
}
|
|
357
|
-
}
|
|
358
|
-
if (scope === "all" || scope === "reference") {
|
|
359
|
-
for (const id of await listReferenceDocs(root)) {
|
|
360
|
-
const stem = id.slice("Reference/".length);
|
|
361
|
-
docs.push({ id, stem, path: join(referenceDir(root), `${stem}.md`) });
|
|
362
|
-
}
|
|
363
|
-
}
|
|
364
|
-
if (docs.length === 0) return { summaries: [], hits: [] };
|
|
365
|
-
|
|
366
|
-
// Split into terms and scan each independently; aggregate per-article so a
|
|
367
|
-
// multi-word query that never appears contiguously still surfaces the
|
|
368
|
-
// articles hitting the most distinct terms. Drop trivial stopwords; if the
|
|
369
|
-
// query is ONLY stopwords, fall back to the whole query as one substring.
|
|
370
|
-
const rawTerms = q.split(/\s+/).filter((t) => t.length > 0);
|
|
371
|
-
const filtered = rawTerms.filter((t) => !STOPWORDS.has(t.toLowerCase()));
|
|
372
|
-
const terms = filtered.length > 0 ? filtered : [q];
|
|
373
|
-
const isMulti = terms.length > 1;
|
|
374
|
-
|
|
375
|
-
const summaries: ArticleSearchSummary[] = [];
|
|
376
|
-
type HitWithCount = SearchHit & { _termCount: number };
|
|
377
|
-
const allHits: HitWithCount[] = [];
|
|
378
|
-
|
|
379
|
-
for (const doc of docs.sort((a, b) => a.id.localeCompare(b.id))) {
|
|
380
|
-
const stem = doc.id;
|
|
381
|
-
let content: string;
|
|
382
|
-
try {
|
|
383
|
-
content = await readFile(doc.path, "utf8");
|
|
384
|
-
} catch {
|
|
385
|
-
continue;
|
|
386
|
-
}
|
|
387
|
-
|
|
388
|
-
const matchedTerms: string[] = [];
|
|
389
|
-
let totalOccurrences = 0;
|
|
390
|
-
for (const t of terms) {
|
|
391
|
-
const n = countSubstringOccurrences(content, t, caseSensitive);
|
|
392
|
-
if (n > 0) {
|
|
393
|
-
matchedTerms.push(t);
|
|
394
|
-
totalOccurrences += n;
|
|
395
|
-
}
|
|
396
|
-
}
|
|
397
|
-
if (totalOccurrences === 0) continue;
|
|
398
|
-
|
|
399
|
-
// Title/stem match bonus: a term in the stem signals the article is *about*
|
|
400
|
-
// that term, not just mentioning it (secondary sort key).
|
|
401
|
-
const stemForMatch = caseSensitive ? stem : stem.toLowerCase();
|
|
402
|
-
let titleMatches = 0;
|
|
403
|
-
for (const t of terms) {
|
|
404
|
-
if (stemForMatch.includes(caseSensitive ? t : t.toLowerCase())) titleMatches++;
|
|
405
|
-
}
|
|
406
|
-
|
|
407
|
-
const summary: ArticleSearchSummary = { article: stem, occurrences: totalOccurrences };
|
|
408
|
-
if (isMulti) summary.matched_terms = matchedTerms;
|
|
409
|
-
if (titleMatches > 0) summary.title_matches = titleMatches;
|
|
410
|
-
summaries.push(summary);
|
|
411
|
-
|
|
412
|
-
const lines = content.split(/\r?\n/);
|
|
413
|
-
let inFence = false;
|
|
414
|
-
let lastAnchor: string | null = null;
|
|
415
|
-
const lcTerms = caseSensitive ? matchedTerms : matchedTerms.map((t) => t.toLowerCase());
|
|
416
|
-
for (let i = 0; i < lines.length; i++) {
|
|
417
|
-
const line = lines[i]!;
|
|
418
|
-
if (isFenceLine(line)) {
|
|
419
|
-
inFence = !inFence;
|
|
420
|
-
} else if (!inFence) {
|
|
421
|
-
const h = parseHeadingLine(line);
|
|
422
|
-
if (h) lastAnchor = slugifyHeading(h.title);
|
|
423
|
-
}
|
|
424
|
-
const compareLine = caseSensitive ? line : line.toLowerCase();
|
|
425
|
-
let termCount = 0;
|
|
426
|
-
for (const t of lcTerms) if (compareLine.includes(t)) termCount++;
|
|
427
|
-
if (termCount > 0) {
|
|
428
|
-
allHits.push({ article: stem, anchor: lastAnchor, line: i + 1, excerpt: line.trim(), _termCount: termCount });
|
|
429
|
-
}
|
|
430
|
-
}
|
|
431
|
-
}
|
|
432
|
-
|
|
433
|
-
// Rank summaries: distinct terms (desc) → title matches (desc) →
|
|
434
|
-
// occurrences (desc) → name (asc, stable).
|
|
435
|
-
summaries.sort((a, b) => {
|
|
436
|
-
const am = a.matched_terms?.length ?? 1;
|
|
437
|
-
const bm = b.matched_terms?.length ?? 1;
|
|
438
|
-
if (bm !== am) return bm - am;
|
|
439
|
-
const at = a.title_matches ?? 0;
|
|
440
|
-
const bt = b.title_matches ?? 0;
|
|
441
|
-
if (bt !== at) return bt - at;
|
|
442
|
-
if (b.occurrences !== a.occurrences) return b.occurrences - a.occurrences;
|
|
443
|
-
return a.article.localeCompare(b.article);
|
|
444
|
-
});
|
|
445
|
-
|
|
446
|
-
allHits.sort((a, b) => b._termCount - a._termCount || a.article.localeCompare(b.article) || a.line - b.line);
|
|
447
|
-
const hits: SearchHit[] = allHits.slice(0, limit).map(({ _termCount, ...h }) => h);
|
|
448
|
-
return { summaries, hits };
|
|
449
|
-
}
|
|
450
|
-
|
|
451
|
-
// ---- setup / status ---------------------------------------------------
|
|
452
|
-
|
|
453
|
-
const README = `# Your memory
|
|
454
|
-
|
|
455
|
-
This is your personal wiki — a structured, durable record of your thinking,
|
|
456
|
-
synthesized from your conversations with your local AI. It belongs to you: plain
|
|
457
|
-
Markdown, version-controlled with git, readable and editable with any tool
|
|
458
|
-
(Obsidian opens this folder as a vault with no setup).
|
|
459
|
-
|
|
460
|
-
mlx-bun curates it; you own it. Edit any file directly — the next synthesis run
|
|
461
|
-
respects your changes.
|
|
462
|
-
|
|
463
|
-
## Structure
|
|
464
|
-
|
|
465
|
-
- \`articles/\` — the wiki articles themselves (\`Topic_Name.md\`)
|
|
466
|
-
- \`Reference/\` — read-only symlinks to mlx-bun's own docs; synthesis never
|
|
467
|
-
writes these
|
|
468
|
-
- \`Meta/\` — operational pages: editorial guidelines, conventions, the
|
|
469
|
-
bucket taxonomy, topics to ignore
|
|
470
|
-
- \`Talk/\` — per-article discussion / conflict notes
|
|
471
|
-
|
|
472
|
-
Nothing here ever leaves your machine.
|
|
473
|
-
`;
|
|
474
|
-
|
|
475
|
-
const EDITORIAL_GUIDELINES = `# Editorial Guidelines
|
|
476
|
-
|
|
477
|
-
This wiki follows Wikipedia's editorial conventions except as noted below. When
|
|
478
|
-
in doubt, do what Wikipedia would do.
|
|
479
|
-
|
|
480
|
-
## Scope adaptations for a personal wiki
|
|
481
|
-
|
|
482
|
-
- **Subject**: the user (you). Articles describe the people, projects, ideas,
|
|
483
|
-
tools, and topics relevant to your thinking.
|
|
484
|
-
- **Sources**: your conversation transcripts with the local assistant. Each
|
|
485
|
-
article cites the conversations that contributed to it (\`conv:HASH\`).
|
|
486
|
-
- **Neutral point of view**: adapted. The wiki represents your actual views and
|
|
487
|
-
reasoning. "Neutral" here means faithful to the evidence in your
|
|
488
|
-
conversations, not pretending to be from no perspective.
|
|
489
|
-
- **Notability**: anything that recurs in your thinking is notable enough. Stub
|
|
490
|
-
articles are encouraged for emerging topics. There is no ownership or
|
|
491
|
-
usefulness test — a subject earns an article by being something you actually
|
|
492
|
-
think about, not by being something you own.
|
|
493
|
-
- **Conflict of interest**: not applicable — this is a single-subject wiki you
|
|
494
|
-
own.
|
|
495
|
-
|
|
496
|
-
## Articles are subjects
|
|
497
|
-
|
|
498
|
-
Each article is named for ONE specific subject — the actual person, project,
|
|
499
|
-
idea, tool, or topic (one concrete named subject per article, drawn from the
|
|
500
|
-
conversation — never a generic placeholder), never a mixed \`X_and_Y\` topic bin. When a chunk is about several subjects it fans out to
|
|
501
|
-
each subject's article, and routing (see \`Bucketing.md\`) strongly prefers an
|
|
502
|
-
existing subject article over minting a new one. This subject-per-article shape
|
|
503
|
-
is the one deliberate improvement over the source wiki.
|
|
504
|
-
|
|
505
|
-
## Article structure
|
|
506
|
-
|
|
507
|
-
See \`Article_Conventions.md\`.
|
|
508
|
-
|
|
509
|
-
## Runs on local AI
|
|
510
|
-
|
|
511
|
-
Synthesis runs entirely on a small local model. Every editorial judgment is
|
|
512
|
-
decomposed into the smallest single-purpose task — ideally a binary ("does this
|
|
513
|
-
chunk update this section? yes/no") — so the model stays within its competence
|
|
514
|
-
and the work is cheap to repeat. The policy is the same one a cloud editor would
|
|
515
|
-
follow; only the execution is decomposed.
|
|
516
|
-
|
|
517
|
-
## Maintenance disposition
|
|
518
|
-
|
|
519
|
-
Synthesis operates as a Wikipedia editor making small, conservative
|
|
520
|
-
contributions. Edits integrate rather than replace. Trajectories ("used to think
|
|
521
|
-
X, now thinks Y") are preserved. Talk pages surface concerns rather than silently
|
|
522
|
-
resolving them.
|
|
523
|
-
|
|
524
|
-
## Time and history
|
|
525
|
-
|
|
526
|
-
Follow Wikipedia's treatment. The article describes the **current state** —
|
|
527
|
-
present tense, in the lead and body. The lead is a concise current-state summary:
|
|
528
|
-
never scatter "previously …" caveats through it.
|
|
529
|
-
|
|
530
|
-
When the user's position genuinely EVOLVES over time, the trajectory is
|
|
531
|
-
**PRESERVED** in a dedicated **## History** section (we KEEP the history — we do
|
|
532
|
-
NOT delete it). The lead/body resolve to the current state; the History section
|
|
533
|
-
records the change with citations to BOTH the earlier and the later source
|
|
534
|
-
(*"previously <earlier>; as of <date> <later>"*).
|
|
535
|
-
|
|
536
|
-
Classify before editing — keep the evolution of the USER'S POSITIONS, not the
|
|
537
|
-
AI's MISTAKES:
|
|
538
|
-
- **Evolution** (a relationship / opinion / preference / stance the user changed
|
|
539
|
-
across dated statements) → resolve the lead to the current value AND add or
|
|
540
|
-
extend the **## History** section preserving the trajectory.
|
|
541
|
-
- **Factual error** (a world-fact or spec that was simply WRONG, or a flat "no,
|
|
542
|
-
that is wrong, it is actually <value>") → **silently overwrite** with the
|
|
543
|
-
correct value: NO History entry, the wrong value pruned entirely as if it had
|
|
544
|
-
never been there.
|
|
545
|
-
`;
|
|
546
|
-
|
|
547
|
-
const BUCKETING = `# Bucketing Policy
|
|
548
|
-
|
|
549
|
-
How a chunk's subjects are routed to articles. This documents the **current
|
|
550
|
-
default behavior** — the pipeline already works this way; edit this page to
|
|
551
|
-
change it.
|
|
552
|
-
|
|
553
|
-
## Defaults
|
|
554
|
-
|
|
555
|
-
- **Strongly prefer existing articles.** A chunk's subject that matches an
|
|
556
|
-
existing article (by canonical name or alias) folds into that article. Only
|
|
557
|
-
mint a NEW subject article when no existing one is a reasonable fit — and even
|
|
558
|
-
then, only when the subject is substantive enough to warrant its own article
|
|
559
|
-
(it recurs in your thinking, or the chunk engages it as a genuine subject),
|
|
560
|
-
not a single fleeting mention.
|
|
561
|
-
- A chunk may belong to **multiple** existing articles when it genuinely spans
|
|
562
|
-
them — it fans out to every subject it is about.
|
|
563
|
-
- If a subject fits one or more existing articles, do not also mint a new article
|
|
564
|
-
for it. **Existing articles win in any tie.**
|
|
565
|
-
- Article names are Wikipedia-style: the specific subject, title-cased (see
|
|
566
|
-
\`Entities.md\`).
|
|
567
|
-
- A subject too thin or fleeting to deserve its own article — and with no
|
|
568
|
-
existing home — is **left captured in its chunk** (still retrievable by
|
|
569
|
-
search), never dropped and never forced into a junk article. It earns an
|
|
570
|
-
article later if it recurs.
|
|
571
|
-
|
|
572
|
-
## How to tune
|
|
573
|
-
|
|
574
|
-
- **More, smaller articles:** lower the bar for minting a new article (split
|
|
575
|
-
readily; treat narrower subjects as article-worthy).
|
|
576
|
-
- **Fewer, larger articles:** raise the bar; prefer folding into an existing
|
|
577
|
-
broad subject over spawning a new one.
|
|
578
|
-
|
|
579
|
-
The router follows this page over its built-in defaults.
|
|
580
|
-
`;
|
|
581
|
-
|
|
582
|
-
const ARTICLE_CONVENTIONS = `# Article Conventions
|
|
583
|
-
|
|
584
|
-
Every article is one Markdown file describing one entity, written to a fixed
|
|
585
|
-
top-to-bottom grammar. The write pipeline emits this shape and the parser reads
|
|
586
|
-
it deterministically — keep the order exactly as below.
|
|
587
|
-
|
|
588
|
-
## Article grammar (fixed, top to bottom)
|
|
589
|
-
|
|
590
|
-
1. **\`# H1\`** — the title, equal to the filename stem with \`_\` → space (the
|
|
591
|
-
file \`<Subject_Name>.md\` has \`# <Subject Name>\`). Exactly one H1.
|
|
592
|
-
2. **Series banner** (optional) — a single italic line directly under the H1:
|
|
593
|
-
\`*Part of a series on [[Series Name]].*\`. Present only when the article belongs
|
|
594
|
-
to a named series.
|
|
595
|
-
3. **Infobox** (optional) — a fenced \`\`\`info\` block of \`key: value\` lines (see
|
|
596
|
-
below). Comes before the lead.
|
|
597
|
-
4. **Lead** — a 2–4 sentence abstract: present tense, names the thing in **bold**
|
|
598
|
-
in the first sentence, says what it is, and characterizes the user's
|
|
599
|
-
relationship to it (uses / owns / chose / considered) — written in the THIRD
|
|
600
|
-
person, not just a spec definition.
|
|
601
|
-
5. **Sections** — \`##\` (and nested \`###\`) sections in natural reading order,
|
|
602
|
-
one sub-topic each.
|
|
603
|
-
6. **\`## See also\`** — bullet list of \`[[wikilinks]]\` to related articles.
|
|
604
|
-
7. **\`## Notes\`** (optional) — \`[^N]\` commentary footnotes.
|
|
605
|
-
8. **\`## References\`** — the \`[^N]:\` definitions. This is ALWAYS the LAST section.
|
|
606
|
-
|
|
607
|
-
## Voice and scope
|
|
608
|
-
|
|
609
|
-
Follow Wikipedia's conventions in every respect EXCEPT two deliberate carve-outs:
|
|
610
|
-
|
|
611
|
-
- **Voice — THIRD PERSON, always.** Write like Wikipedia: never the first person ("I
|
|
612
|
-
use…", "my pick"). Refer to the user in the third person ("the user uses…",
|
|
613
|
-
"the user considers…"). The subject of the article is the topic; the article
|
|
614
|
-
records what the user thinks and does regarding it.
|
|
615
|
-
- **Carve-out 1 — notability.** Wikipedia's notability bar does NOT apply: anything
|
|
616
|
-
the user discussed is notable enough for an article (see \`Entities.md\`).
|
|
617
|
-
- **Carve-out 2 — neutrality.** Wikipedia's neutral-point-of-view does NOT apply:
|
|
618
|
-
these articles document the user's OWN positions and opinions. Capture the user's
|
|
619
|
-
current stance on the topic accurately, including subjective judgments — the goal
|
|
620
|
-
is to record what the user currently thinks. Everything else (structure, citations,
|
|
621
|
-
summary style, third-person voice) is standard Wikipedia.
|
|
622
|
-
|
|
623
|
-
## Citations / footnotes
|
|
624
|
-
|
|
625
|
-
Inline claims carry a \`[^N]\` marker; the \`## References\` section collects the
|
|
626
|
-
matching definitions. Every marker has exactly one definition and vice-versa.
|
|
627
|
-
|
|
628
|
-
The footnote definition format is fixed (the Lucien standard):
|
|
629
|
-
|
|
630
|
-
\`\`\`
|
|
631
|
-
[^N]: \`conv:HASH\` (YYYY-MM-DD, source) — desc
|
|
632
|
-
\`\`\`
|
|
633
|
-
|
|
634
|
-
where **\`HASH\` = the first 8 lowercase hex digits of the conversation UUID with
|
|
635
|
-
hyphens stripped** (e.g. \`00000000-0000-0000-0000-000000000000\` → \`00000000\`).
|
|
636
|
-
The hash identifies the CONVERSATION only — it deliberately carries no chunk
|
|
637
|
-
range — so chunk→section provenance is recovered from structured state, never by
|
|
638
|
-
parsing footnotes. \`desc\` is a short note of what the conversation covered.
|
|
639
|
-
|
|
640
|
-
## Links
|
|
641
|
-
|
|
642
|
-
Use wikilinks for internal references: \`[[Article Name]]\`. These resolve to the
|
|
643
|
-
correct files (spaces vs underscores and case are normalized).
|
|
644
|
-
|
|
645
|
-
## Infobox
|
|
646
|
-
|
|
647
|
-
The infobox is a fenced \`\`\`info\` block placed between the H1/banner and the
|
|
648
|
-
lead. It is fenced so \`parseToc\`/\`extractSection\`/\`searchArticles\` skip it (its
|
|
649
|
-
keys are never headings or searchable prose). Rules:
|
|
650
|
-
|
|
651
|
-
- Keys are \`snake_case\`.
|
|
652
|
-
- A value that names another entity is a \`[[wikilink]]\` (load-bearing for
|
|
653
|
-
NAVIGATION — it feeds the link graph).
|
|
654
|
-
- \`aliases:\` is a **comma**-separated list feeding the entity index.
|
|
655
|
-
- Other multi-value fields are **\`;\`**-separated.
|
|
656
|
-
- A \`kind:\` field tags the closed entity-kind (\`thing\`/\`person\`/\`domain\`/
|
|
657
|
-
\`project\`/\`standard\`, default \`thing\`); a \`type:\` field carries the emergent
|
|
658
|
-
label (a short free-form noun phrase). See \`Infobox_Schemas.md\`.
|
|
659
|
-
|
|
660
|
-
Every field is either a **world-fact** (spec-sheet / citable) or a
|
|
661
|
-
**relationship-fact** (conversation-established — the personal layer). Both are
|
|
662
|
-
CONTENT the AI reads; there is no numeric facet or \`gte/lte\` query over any field.
|
|
663
|
-
|
|
664
|
-
### Field-kind taxonomy
|
|
665
|
-
|
|
666
|
-
The table below teaches the value SHAPE per field-kind only — every \`field\` cell
|
|
667
|
-
is a \`<…>\` placeholder and every \`example\` cell is schematic. NEVER copy a key
|
|
668
|
-
or a value from here into an article; every key and value must come from the
|
|
669
|
-
chunk text about the actual entity. The fixed keys (\`type\`, \`kind\`, \`owned\`,
|
|
670
|
-
\`acquired\`, \`used_for\`, \`chosen_over\`, \`status\`, \`opinion\`, \`aliases\`) are the
|
|
671
|
-
domain-neutral vocabulary to draw from when the chunks support them; every
|
|
672
|
-
domain-specific key is supplied by the chunk, never by this page.
|
|
673
|
-
|
|
674
|
-
| field | kind | value-type | example (schematic) |
|
|
675
|
-
| --- | --- | --- | --- |
|
|
676
|
-
| \`type\` | world | enum | \`<entity type>\` |
|
|
677
|
-
| \`kind\` | world | enum | one of thing\\|person\\|domain\\|project\\|standard |
|
|
678
|
-
| \`<world_entity_key>\` | world | entity-link | \`[[Linked Entity]]\` |
|
|
679
|
-
| \`<world_scalar_key>\` | world | scalar | \`<value>\` |
|
|
680
|
-
| \`<world_enum_key>\` | world | enum | \`<value>\` |
|
|
681
|
-
| \`<world_list_key>\` | world | list | \`[[Entity A]]; [[Entity B]]\` |
|
|
682
|
-
| \`owned\` | relationship | enum | \`yes\` (\`yes\`\\|\`no\`\\|\`wishlist\`) |
|
|
683
|
-
| \`acquired\` | relationship | date | \`<YYYY-MM + how>\` |
|
|
684
|
-
| \`used_for\` | relationship | list | \`<use>; <use>\` |
|
|
685
|
-
| \`chosen_over\` | relationship | entity-link | \`[[Linked Entity]]\` |
|
|
686
|
-
| \`status\` | relationship | scalar | \`<short status>\` |
|
|
687
|
-
| \`opinion\` | relationship | scalar | \`<your one-line take>\` |
|
|
688
|
-
|
|
689
|
-
\`owned\`'s \`wishlist\` value means wants-but-doesn't-yet-own. Each world-fact is
|
|
690
|
-
spec-sheet verifiable; each relationship-fact is established in conversation.
|
|
691
|
-
Apply a field only when the chunk supports it — a world-fact spec key belongs on
|
|
692
|
-
the kind of entity it describes (typically a hardware \`thing\`), never on a
|
|
693
|
-
\`domain\` or \`standard\`.
|
|
694
|
-
|
|
695
|
-
### Example info block (ILLUSTRATIVE SCHEMA — never copy these values)
|
|
696
|
-
|
|
697
|
-
The block below is a SHAPE, not content: every \`<…>\` is a placeholder. NEVER
|
|
698
|
-
copy these keys or values into a real article — emit ONLY facts the article's own
|
|
699
|
-
source chunks establish, and OMIT any key the chunks do not support. A key that
|
|
700
|
-
does not apply to the entity (e.g. a world-fact spec key on a \`domain\`, \`person\`,
|
|
701
|
-
or \`standard\`) must not appear at all.
|
|
702
|
-
|
|
703
|
-
\`\`\`info
|
|
704
|
-
type: <emergent label, from the chunks>
|
|
705
|
-
kind: <thing | person | domain | project | standard>
|
|
706
|
-
<world_fact_key>: <world value, from the chunks>
|
|
707
|
-
<relationship_fact_key>: <relationship value, from the chunks>
|
|
708
|
-
aliases: <comma-separated surface variants>
|
|
709
|
-
\`\`\`
|
|
710
|
-
|
|
711
|
-
## Categories
|
|
712
|
-
|
|
713
|
-
An article declares its category memberships with a single line near the top:
|
|
714
|
-
|
|
715
|
-
\`\`\`
|
|
716
|
-
categories: [[Category:<Name>]], [[Category:<Name>]]
|
|
717
|
-
\`\`\`
|
|
718
|
-
|
|
719
|
-
Categories are emergent and explicit — declared per-article by judgment, never
|
|
720
|
-
derived from the infobox \`type\`. See \`Category_Definitions.md\`.
|
|
721
|
-
|
|
722
|
-
## Stubs
|
|
723
|
-
|
|
724
|
-
New or thin articles may be marked \`{{stub}}\` at the top. An article is a stub
|
|
725
|
-
when it has fewer than 2 body sections OR fewer than 40 words of body prose
|
|
726
|
-
(excluding the infobox and References). Stubs are legitimate — depth comes
|
|
727
|
-
through accumulation. See \`Summary_Style.md\` for split/merge rules.
|
|
728
|
-
`;
|
|
729
|
-
|
|
730
|
-
const BUCKETS = `# Buckets
|
|
731
|
-
|
|
732
|
-
> **Superseded by \`Entities.md\` for synthesis targeting; retained as optional
|
|
733
|
-
> staging.** Synthesis now routes chunks to entity articles (see \`Entities.md\`),
|
|
734
|
-
> not buckets. Buckets survive only as an optional staging layer.
|
|
735
|
-
|
|
736
|
-
The bucket taxonomy used to organize segments of conversation into articles.
|
|
737
|
-
Buckets are emergent — derived from the actual content of conversations, not
|
|
738
|
-
imposed in advance. Populated by synthesis runs.
|
|
739
|
-
`;
|
|
740
|
-
|
|
741
|
-
const TOPICS_TO_IGNORE = `# Topics to Ignore
|
|
742
|
-
|
|
743
|
-
The chunker consults this page (mandatorily) before emitting chunks.
|
|
744
|
-
|
|
745
|
-
**Current policy: nothing is ignored. Surface everything.** Emit chunks for all substantive content; do not omit any span on the basis of this page. There are no excluded topics, people, or content categories.
|
|
746
|
-
|
|
747
|
-
To change this later: add plain-text descriptions of topics/people/content that should not be synthesized into the Dreaming, one per line under a list below. Semantic matching catches related content even when phrasing differs. While this section says "nothing is ignored," ignore nothing.
|
|
748
|
-
`;
|
|
749
|
-
|
|
750
|
-
const INFOBOX_SCHEMAS = `# Infobox Schemas
|
|
751
|
-
|
|
752
|
-
Infobox schemas are **emergent**, not a closed type list. This page is the
|
|
753
|
-
editable policy: a fixed generic grammar plus a growing registry of per-\`type\`
|
|
754
|
-
field sets the model has actually used. There is NO code-level enum of infobox
|
|
755
|
-
\`type\`s and NO \`type→schema\` map — editing this page re-tunes infobox extraction
|
|
756
|
-
with no code change.
|
|
757
|
-
|
|
758
|
-
## The grammar (the only fixed part)
|
|
759
|
-
|
|
760
|
-
1. **Every fact is one of two kinds:**
|
|
761
|
-
- a **world-fact** — spec-sheet / citable (a \`<world_fact_key>\`);
|
|
762
|
-
- a **relationship-fact** — conversation-established, the personal-notability
|
|
763
|
-
layer (\`owned\`, \`used_for\`, \`chosen_over\`, and the like).
|
|
764
|
-
Both are CONTENT the AI reads; neither is a query/filter index.
|
|
765
|
-
2. **Keys are \`snake_case\`.**
|
|
766
|
-
3. **A value naming another entity is a \`[[wikilink]]\`** — this feeds the link
|
|
767
|
-
graph for navigation.
|
|
768
|
-
4. **Value-types are \`scalar | entity-link | enum | list | date\`.**
|
|
769
|
-
5. **\`kind:\`** tags the closed entity-kind (\`thing\`/\`person\`/\`domain\`/
|
|
770
|
-
\`project\`/\`standard\`, default \`thing\`); **\`type:\`** carries the emergent label
|
|
771
|
-
(a short free-form label).
|
|
772
|
-
|
|
773
|
-
When a newly-emergent \`type\` has no registry entry, propose one (a world /
|
|
774
|
-
relationship field split) and append it below. Reconcile a new \`type\` exactly as
|
|
775
|
-
a new category name is added to \`Category_Definitions.md\`.
|
|
776
|
-
|
|
777
|
-
## Registry (grows by judgment)
|
|
778
|
-
|
|
779
|
-
Each entry below is an **illustrative seed, not a closed set — extend by judgment
|
|
780
|
-
when a new kind of thing appears.** Field NAMES are a vocabulary to draw from
|
|
781
|
-
WHEN the chunks support them; any example VALUE anywhere in Meta is illustrative
|
|
782
|
-
ONLY. **NEVER copy a key or value (a \`type\`, a \`<world_fact_key>\`) into an article
|
|
783
|
-
whose own chunks do not establish it**, and never attach a world-fact spec to a
|
|
784
|
-
\`domain\` / \`person\` / \`standard\` entity.
|
|
785
|
-
|
|
786
|
-
### <type>
|
|
787
|
-
*Illustrative SHAPE of a registry entry, not real content — every \`<…>\` is a
|
|
788
|
-
placeholder; the real keys are supplied by the entity's own chunks. Extend by
|
|
789
|
-
judgment.*
|
|
790
|
-
- **world:** \`type\`, \`<world_fact_key>\`, \`<world_fact_key>\`
|
|
791
|
-
- **relationship:** \`owned\`, \`acquired\`, \`used_for\`, \`chosen_over\`, \`opinion\`, \`status\`
|
|
792
|
-
|
|
793
|
-
The fixed relationship keys above are the domain-neutral personal layer; each
|
|
794
|
-
\`<world_fact_key>\` slot is filled per-\`type\` from the chunks, NOT from this page.
|
|
795
|
-
`;
|
|
796
|
-
|
|
797
|
-
const CATEGORY_DEFINITIONS = `# Category Definitions
|
|
798
|
-
|
|
799
|
-
Categories are **emergent**, derived from your actual content — never imposed in
|
|
800
|
-
advance. There is NO hardcoded category vocabulary and NO \`type→category\` map.
|
|
801
|
-
This page is the editable policy plus a growing registry; editing it re-tunes
|
|
802
|
-
categorization with no code change.
|
|
803
|
-
|
|
804
|
-
## What a category is
|
|
805
|
-
|
|
806
|
-
A category is a **Wikipedia-style grouping of things** — a named set an article
|
|
807
|
-
joins by declaring membership. An article declares its categories with a single
|
|
808
|
-
line near the top:
|
|
809
|
-
|
|
810
|
-
\`\`\`
|
|
811
|
-
categories: [[Category:<Name>]], [[Category:<Name>]]
|
|
812
|
-
\`\`\`
|
|
813
|
-
|
|
814
|
-
Categories come into existence the first time an article declares them. A fresh
|
|
815
|
-
wiki starts with **zero** categories and grows them as articles are written.
|
|
816
|
-
|
|
817
|
-
## Naming rules
|
|
818
|
-
|
|
819
|
-
- **Title-case plural noun** (a \`<Plural Noun>\` naming the shared grouping).
|
|
820
|
-
- **Specific**, not vague (a precise grouping name, never a catch-all like \`Things\`).
|
|
821
|
-
- **Create a new category only when ≥2 articles would genuinely share it**;
|
|
822
|
-
otherwise reuse an existing one or omit it. **Existing categories win** —
|
|
823
|
-
prefer an established name over a near-duplicate new one.
|
|
824
|
-
|
|
825
|
-
## Registry
|
|
826
|
-
|
|
827
|
-
One \`Name — definition — parent?\` line is added here as each category is first
|
|
828
|
-
created. (Empty until the first category is declared.)
|
|
829
|
-
`;
|
|
830
|
-
|
|
831
|
-
const ENTITIES_POLICY = `# Entities
|
|
832
|
-
|
|
833
|
-
Governs ENTITY-EXTRACT and ROUTE: which things become entity articles, and how
|
|
834
|
-
they are named. Inlined into the stage prompts.
|
|
835
|
-
|
|
836
|
-
## The five kinds
|
|
837
|
-
|
|
838
|
-
Every entity has exactly one \`kind\` from the closed set:
|
|
839
|
-
|
|
840
|
-
- **thing** — a concrete object, product, or artifact (a named object).
|
|
841
|
-
- **person** — a named individual.
|
|
842
|
-
- **domain** — a broad subject area or field (a named field or area of study).
|
|
843
|
-
- **project** — an ongoing effort with a goal.
|
|
844
|
-
- **standard** — a spec, format, protocol, or material (a named spec or material).
|
|
845
|
-
|
|
846
|
-
Default to \`thing\` when no other kind clearly applies.
|
|
847
|
-
|
|
848
|
-
## Which subjects earn an article
|
|
849
|
-
|
|
850
|
-
This page does NOT gate on ownership, usefulness, or any single hard count.
|
|
851
|
-
Notability is editorial: **anything that recurs in your thinking is notable
|
|
852
|
-
enough**, and stub articles are encouraged for emerging topics (see
|
|
853
|
-
\`Editorial_Guidelines.md\`). Routing strongly prefers an existing article and
|
|
854
|
-
mints a new one only for a substantive, recurring subject with no existing home
|
|
855
|
-
(see \`Bucketing.md\`). **Nothing is dropped as trivia** — a subject with no
|
|
856
|
-
article yet is captured in its chunk and stays retrievable by search.
|
|
857
|
-
|
|
858
|
-
## Canonical naming
|
|
859
|
-
|
|
860
|
-
- Use **Wikipedia title casing**.
|
|
861
|
-
- Prefer the **specific subject name** (the actual named subject from the chunk,
|
|
862
|
-
not a generic descriptor like "the thing" or "that one").
|
|
863
|
-
- \`aliases\` capture the variants (spelling, spacing, and short-form variants of
|
|
864
|
-
the same name) so they all resolve to one article.
|
|
865
|
-
|
|
866
|
-
## Emit both granularities
|
|
867
|
-
|
|
868
|
-
When a chunk is about a specific thing inside a broad area, emit **both** the
|
|
869
|
-
specific entity AND the broad domain it belongs to — the precise product/proper
|
|
870
|
-
name and its connective field — so both the precise and the connective article
|
|
871
|
-
get the chunk. Do NOT copy example names from this policy; only emit things that
|
|
872
|
-
actually appear in the chunk text.
|
|
873
|
-
|
|
874
|
-
## Always keep the most specific name
|
|
875
|
-
|
|
876
|
-
ALWAYS emit the **most specific proper noun present in the chunk text** (the named
|
|
877
|
-
product, tool, person, place, or model). Never drop the concrete named thing in
|
|
878
|
-
favor of only the broad domain — the specific name is the load-bearing entity.
|
|
879
|
-
`;
|
|
880
|
-
|
|
881
|
-
const SUMMARY_STYLE = `# Summary Style
|
|
882
|
-
|
|
883
|
-
How articles split, merge, and mark stubs — Wikipedia's summary-style rules,
|
|
884
|
-
adapted. NORMALIZE and the parser recognize the template syntax defined here.
|
|
885
|
-
|
|
886
|
-
## SPLIT
|
|
887
|
-
|
|
888
|
-
When a thing covered inside a section **crosses the notability checklist** AND is
|
|
889
|
-
**large enough** (≥ a screen of prose, or worth its own infobox), move it to its
|
|
890
|
-
own article \`Specific_Thing.md\`. In the parent, leave a **2–3 sentence summary**
|
|
891
|
-
followed by a main-article pointer:
|
|
892
|
-
|
|
893
|
-
\`\`\`
|
|
894
|
-
*Main article: [[Specific_Thing]]*
|
|
895
|
-
\`\`\`
|
|
896
|
-
|
|
897
|
-
Every \`[^N]\` citation in the moved text travels with it to the child — no facts
|
|
898
|
-
are lost.
|
|
899
|
-
|
|
900
|
-
## MERGE
|
|
901
|
-
|
|
902
|
-
When two stubs describe the **same thing under different names**, merge them into
|
|
903
|
-
the canonical title. The loser's name becomes an \`alias\` on the survivor, and a
|
|
904
|
-
redirect note records the merge.
|
|
905
|
-
|
|
906
|
-
## Template syntax
|
|
907
|
-
|
|
908
|
-
- **\`{{Main}}\`** — the main-article pointer, written \`*Main article: [[Target]]*\`.
|
|
909
|
-
- **\`{{stub}}\`** — placed at the top of a thin article.
|
|
910
|
-
- **redirect** — a one-line note on a merged-away title pointing at the canonical
|
|
911
|
-
article (\`*Redirects to [[Canonical]].*\`).
|
|
912
|
-
|
|
913
|
-
## Stub threshold (baked)
|
|
914
|
-
|
|
915
|
-
An article is a **stub** when it has **fewer than 2 body sections OR fewer than
|
|
916
|
-
40 words of body prose** (excluding the infobox and the References section).
|
|
917
|
-
|
|
918
|
-
## Worked example (SCHEMATIC — \`<…>\` placeholders only, never copy these literally)
|
|
919
|
-
|
|
920
|
-
A \`<Parent Subject>\` article has a \`<Child Subject>\` subsection that has grown to
|
|
921
|
-
own an infobox-worth of owned/used facts — it crosses notability and size, so it
|
|
922
|
-
SPLITS.
|
|
923
|
-
|
|
924
|
-
**Parent, after the split** (\`<Parent_Subject>\`, the \`<Child Subject>\` subsection):
|
|
925
|
-
|
|
926
|
-
\`\`\`
|
|
927
|
-
### <Child Subject>
|
|
928
|
-
|
|
929
|
-
<A 2–3 sentence summary of the child subject and how the user relates to it,
|
|
930
|
-
carrying any citation marker(s) from the moved prose.>[^k]
|
|
931
|
-
|
|
932
|
-
*Main article: [[<Child Subject>]]*
|
|
933
|
-
\`\`\`
|
|
934
|
-
|
|
935
|
-
**New child** (\`<Child_Subject>.md\`), with its own infobox and the facts moved
|
|
936
|
-
from the parent:
|
|
937
|
-
|
|
938
|
-
- \`# <Child Subject>\`
|
|
939
|
-
- an \`info\` block: \`type: <emergent label>\`, \`<world_fact_key>: [[Linked Entity]]\`,
|
|
940
|
-
\`<world_fact_key>: <value>\`, \`kind: thing\`, \`owned: <yes|no|wishlist>\`,
|
|
941
|
-
\`acquired: <YYYY-MM + how>\`,
|
|
942
|
-
\`used_for: <use>; <use>\`,
|
|
943
|
-
\`aliases: <surface variant>, <surface variant>\`
|
|
944
|
-
- a lead naming the **<Child Subject>** in bold, then \`## See also\` →
|
|
945
|
-
\`[[Linked Entity]]\`, then \`## References\` last.
|
|
946
|
-
|
|
947
|
-
Every \`[^N]\` from the parent subsection (here \`[^k]\`) travels to the child — the
|
|
948
|
-
\`[^k]: \`conv:HASH\` (…)\` definition moves with the prose so no citation is lost.
|
|
949
|
-
`;
|
|
950
|
-
|
|
951
|
-
const CHUNKING = `# Chunking Style
|
|
952
|
-
|
|
953
|
-
How the conversation-segmentation stage decides what a chunk is. This documents the **current default behavior** — the pipeline already works this way; edit this page to change it. (The mechanism — every conversation is segmented into topic chunks emitted in the fixed output schema — is not tunable here; only the style below is.)
|
|
954
|
-
|
|
955
|
-
## Defaults
|
|
956
|
-
|
|
957
|
-
- Most substantive conversations contain **multiple** chunks, not one. Look hard for topic shifts.
|
|
958
|
-
- A topic shift is when the discussion moves from one subject to a **substantially different** subject (e.g. one feature → a different feature; a problem → a distinct related concern; topic X → topic Y).
|
|
959
|
-
- Sub-chunks for specific aspects within a single broad topic are acceptable.
|
|
960
|
-
- A short conversation (under ~4 messages, a single Q&A) may legitimately be one chunk. Most longer conversations are multiple.
|
|
961
|
-
- A topic that recurs later in the same conversation is its **own separate chunk**, not merged with the earlier occurrence.
|
|
962
|
-
- **Overlap:** two chunks may share **at most one boundary message**, and only when that message is genuinely substantive to *both* the topic ending and the one beginning (true dual-membership). Never overlap for connective filler ("anyway, switching gears"), and never let one chunk's range envelop several messages of another's — if two chunks would overlap by more than one message, redraw the boundary instead.
|
|
963
|
-
- Labels are specific and descriptive, ~4–10 words: name the concrete subject and the facet discussed (\`<specific subject> <facet>\`), never a vague catch-all (\`<generic topic word>\` alone, "a technical discussion", "a Q&A").
|
|
964
|
-
|
|
965
|
-
## How to tune
|
|
966
|
-
|
|
967
|
-
To bias toward fewer, larger chunks: instruct fewer/coarser boundaries and discourage aspect sub-chunks. To bias toward more, smaller chunks: encourage finer aspect splits. To change overlap behavior: state how aggressively boundary messages should be shared. The chunker follows this page over its built-in defaults.
|
|
968
|
-
`;
|
|
969
|
-
|
|
970
|
-
export const META_PAGES: Record<string, string> = {
|
|
971
|
-
"Editorial_Guidelines.md": EDITORIAL_GUIDELINES,
|
|
972
|
-
"Bucketing.md": BUCKETING,
|
|
973
|
-
"Article_Conventions.md": ARTICLE_CONVENTIONS,
|
|
974
|
-
"Infobox_Schemas.md": INFOBOX_SCHEMAS,
|
|
975
|
-
"Category_Definitions.md": CATEGORY_DEFINITIONS,
|
|
976
|
-
"Entities.md": ENTITIES_POLICY,
|
|
977
|
-
"Summary_Style.md": SUMMARY_STYLE,
|
|
978
|
-
"Buckets.md": BUCKETS,
|
|
979
|
-
"Chunking.md": CHUNKING,
|
|
980
|
-
"Topics_to_Ignore.md": TOPICS_TO_IGNORE,
|
|
981
|
-
};
|
|
982
|
-
|
|
983
|
-
const REPO_ROOT = join(import.meta.dir, "..", "..");
|
|
984
|
-
|
|
985
|
-
const REFERENCE_LINKS: { name: string; source: string }[] = [
|
|
986
|
-
{ name: "mlx-bun_README", source: join(REPO_ROOT, "README.md") },
|
|
987
|
-
{ name: "mlx-bun_Server_API", source: join(REPO_ROOT, "docs", "reference", "server-api.md") },
|
|
988
|
-
{ name: "mlx-bun_Server_Config", source: join(REPO_ROOT, "docs", "reference", "server-config.md") },
|
|
989
|
-
{ name: "mlx-bun_Library_API", source: join(REPO_ROOT, "docs", "reference", "library-api.md") },
|
|
990
|
-
{ name: "mlx-bun_Distribution", source: join(REPO_ROOT, "docs", "reference", "distribution.md") },
|
|
991
|
-
{ name: "mlx-bun_Training", source: join(REPO_ROOT, "docs", "reference", "training.md") },
|
|
992
|
-
{ name: "mlx-bun_Product_Roadmap", source: join(REPO_ROOT, "docs", "planning", "PRODUCT_ROADMAP.md") },
|
|
993
|
-
];
|
|
994
|
-
|
|
995
|
-
async function seedReferenceLinks(root: string): Promise<string[]> {
|
|
996
|
-
await mkdir(referenceDir(root), { recursive: true });
|
|
997
|
-
const changed: string[] = [];
|
|
998
|
-
for (const ref of REFERENCE_LINKS) {
|
|
999
|
-
if (!(await pathExists(ref.source))) continue;
|
|
1000
|
-
const dest = join(referenceDir(root), `${ref.name}.md`);
|
|
1001
|
-
try {
|
|
1002
|
-
const existing = await lstat(dest);
|
|
1003
|
-
if (existing.isSymbolicLink()) {
|
|
1004
|
-
if ((await readlink(dest)) === ref.source) continue;
|
|
1005
|
-
await unlink(dest);
|
|
1006
|
-
} else {
|
|
1007
|
-
continue; // never clobber a user's real file
|
|
1008
|
-
}
|
|
1009
|
-
} catch {
|
|
1010
|
-
// Missing: create it below.
|
|
1011
|
-
}
|
|
1012
|
-
await symlink(ref.source, dest);
|
|
1013
|
-
changed.push(dest);
|
|
1014
|
-
}
|
|
1015
|
-
return changed;
|
|
1016
|
-
}
|
|
1017
|
-
|
|
1018
|
-
function runGit(args: string[], cwd: string): Promise<boolean> {
|
|
1019
|
-
return new Promise((resolve) => {
|
|
1020
|
-
const proc = spawn("git", args, { cwd, stdio: "ignore" });
|
|
1021
|
-
proc.on("exit", (code) => resolve(code === 0));
|
|
1022
|
-
proc.on("error", () => resolve(false));
|
|
1023
|
-
});
|
|
1024
|
-
}
|
|
1025
|
-
|
|
1026
|
-
// ---- read-only git plumbing (per-article history/diff) ----------------
|
|
1027
|
-
//
|
|
1028
|
-
// Used by the memory REST surface (GET /api/memory/history|diff) to render
|
|
1029
|
-
// "watch it self-heal" — the article's commit history and a specific
|
|
1030
|
-
// commit's diff. Read-only plumbing only (log/show), args always an
|
|
1031
|
-
// explicit argv array via Bun.spawn (never a shell string), and callers
|
|
1032
|
-
// MUST validate `rev`/`name` before calling these (see server.ts) — this
|
|
1033
|
-
// layer trusts its `relPath`/`rev` inputs are already safe.
|
|
1034
|
-
|
|
1035
|
-
export interface ArticleHistoryEntry {
|
|
1036
|
-
hash: string;
|
|
1037
|
-
date: string;
|
|
1038
|
-
subject: string;
|
|
1039
|
-
}
|
|
1040
|
-
|
|
1041
|
-
/** `git log` for one file, newest first, argv-only (never a shell string).
|
|
1042
|
-
* Returns [] when the vault isn't a git repo, the file has no history, or
|
|
1043
|
-
* git is unavailable. `relPath` must already be a repo-relative path (no
|
|
1044
|
-
* `..`, not absolute) — callers validate before calling. */
|
|
1045
|
-
export async function articleHistory(root: string, relPath: string, limit = 50): Promise<ArticleHistoryEntry[]> {
|
|
1046
|
-
if (!(await pathExists(join(root, ".git")))) return [];
|
|
1047
|
-
try {
|
|
1048
|
-
const proc = Bun.spawn(
|
|
1049
|
-
["git", "log", `--max-count=${limit}`, "--date=short", "--pretty=format:%H%x1f%ad%x1f%s%x1e", "--", relPath],
|
|
1050
|
-
{ cwd: root, stdout: "pipe", stderr: "ignore" },
|
|
1051
|
-
);
|
|
1052
|
-
const out = await new Response(proc.stdout).text();
|
|
1053
|
-
if ((await proc.exited) !== 0) return [];
|
|
1054
|
-
return out
|
|
1055
|
-
.split("\x1e")
|
|
1056
|
-
.map((rec) => rec.trim())
|
|
1057
|
-
.filter((rec) => rec.length > 0)
|
|
1058
|
-
.map((rec) => {
|
|
1059
|
-
const [hash = "", date = "", subject = ""] = rec.split("\x1f");
|
|
1060
|
-
return { hash, date, subject };
|
|
1061
|
-
})
|
|
1062
|
-
.filter((e) => e.hash.length > 0);
|
|
1063
|
-
} catch {
|
|
1064
|
-
return [];
|
|
1065
|
-
}
|
|
1066
|
-
}
|
|
1067
|
-
|
|
1068
|
-
/** `git show <rev> -- <relPath>` — the diff for one commit touching one file.
|
|
1069
|
-
* `rev` must already be validated as a hex commit hash by the caller (never
|
|
1070
|
-
* a ref expression like `HEAD~1` or `main^`, which could be abused as an
|
|
1071
|
-
* argument-injection vector); `relPath` must already be repo-relative.
|
|
1072
|
-
* Returns null when the vault isn't a git repo or the commit/diff can't be
|
|
1073
|
-
* produced. */
|
|
1074
|
-
export async function articleDiff(root: string, rev: string, relPath: string): Promise<string | null> {
|
|
1075
|
-
if (!(await pathExists(join(root, ".git")))) return null;
|
|
1076
|
-
try {
|
|
1077
|
-
const proc = Bun.spawn(
|
|
1078
|
-
["git", "show", rev, "--pretty=format:", "--", relPath],
|
|
1079
|
-
{ cwd: root, stdout: "pipe", stderr: "ignore" },
|
|
1080
|
-
);
|
|
1081
|
-
const out = await new Response(proc.stdout).text();
|
|
1082
|
-
if ((await proc.exited) !== 0) return null;
|
|
1083
|
-
return out;
|
|
1084
|
-
} catch {
|
|
1085
|
-
return null;
|
|
1086
|
-
}
|
|
1087
|
-
}
|
|
1088
|
-
|
|
1089
|
-
export interface SetupResult {
|
|
1090
|
-
root: string;
|
|
1091
|
-
created: string[];
|
|
1092
|
-
gitInitialized: boolean;
|
|
1093
|
-
alreadySetUp: boolean;
|
|
1094
|
-
}
|
|
1095
|
-
|
|
1096
|
-
/** Idempotent: create dirs, write README + Meta pages (only if missing — never
|
|
1097
|
-
* clobber user edits), git init + initial commit. Safe to re-run. */
|
|
1098
|
-
export async function setupVault(root = vaultRoot()): Promise<SetupResult> {
|
|
1099
|
-
const created: string[] = [];
|
|
1100
|
-
const ensureDir = async (p: string) => {
|
|
1101
|
-
if (!(await pathExists(p))) {
|
|
1102
|
-
await mkdir(p, { recursive: true });
|
|
1103
|
-
created.push(p);
|
|
1104
|
-
}
|
|
1105
|
-
};
|
|
1106
|
-
const writeIfMissing = async (p: string, content: string) => {
|
|
1107
|
-
if (!(await pathExists(p))) {
|
|
1108
|
-
await writeFile(p, content);
|
|
1109
|
-
created.push(p);
|
|
1110
|
-
}
|
|
1111
|
-
};
|
|
1112
|
-
|
|
1113
|
-
await ensureDir(root);
|
|
1114
|
-
for (const sub of ["articles", "Reference", "Meta", "Talk"]) await ensureDir(join(root, sub));
|
|
1115
|
-
await writeIfMissing(join(root, "README.md"), README);
|
|
1116
|
-
// The vault is pure markdown — the synthesis DB lives in the rebuildable
|
|
1117
|
-
// cache (~/.cache/mlx-bun/memory.sqlite), not here. Just keep OS/editor
|
|
1118
|
-
// cruft (Finder, Obsidian workspace state) out of git history.
|
|
1119
|
-
await writeIfMissing(join(root, ".gitignore"), ".DS_Store\n.obsidian/workspace*\n");
|
|
1120
|
-
for (const [filename, content] of Object.entries(META_PAGES)) {
|
|
1121
|
-
await writeIfMissing(join(root, "Meta", filename), content);
|
|
1122
|
-
}
|
|
1123
|
-
created.push(...(await seedReferenceLinks(root)));
|
|
1124
|
-
|
|
1125
|
-
let gitInitialized = false;
|
|
1126
|
-
if (!(await pathExists(join(root, ".git")))) {
|
|
1127
|
-
if (await runGit(["init"], root)) {
|
|
1128
|
-
await runGit(["add", "."], root);
|
|
1129
|
-
await runGit(["commit", "-m", "Initialize mlx-bun memory wiki"], root);
|
|
1130
|
-
gitInitialized = true;
|
|
1131
|
-
}
|
|
1132
|
-
}
|
|
1133
|
-
|
|
1134
|
-
return { root, created, gitInitialized, alreadySetUp: created.length === 0 && !gitInitialized };
|
|
1135
|
-
}
|
|
1136
|
-
|
|
1137
|
-
/** Stage everything and commit, if the vault is a git repo and has changes.
|
|
1138
|
-
* Best-effort: silently no-ops when git is unavailable or nothing changed. */
|
|
1139
|
-
export async function commitVault(root: string, message: string): Promise<void> {
|
|
1140
|
-
if (!(await pathExists(join(root, ".git")))) return;
|
|
1141
|
-
await runGit(["add", "."], root);
|
|
1142
|
-
await runGit(["commit", "-m", message], root);
|
|
1143
|
-
}
|
|
1144
|
-
|
|
1145
|
-
export interface VaultStatus {
|
|
1146
|
-
root: string;
|
|
1147
|
-
exists: boolean;
|
|
1148
|
-
articleCount: number;
|
|
1149
|
-
referenceCount: number;
|
|
1150
|
-
isGitRepo: boolean;
|
|
1151
|
-
recentArticles: { article: string; mtimeMs: number }[];
|
|
1152
|
-
}
|
|
1153
|
-
|
|
1154
|
-
export async function vaultStatus(root = vaultRoot()): Promise<VaultStatus> {
|
|
1155
|
-
const exists = await pathExists(root);
|
|
1156
|
-
const stems = exists ? await listArticles(root) : [];
|
|
1157
|
-
const refs = exists ? await listReferenceDocs(root) : [];
|
|
1158
|
-
const isGitRepo = exists && (await pathExists(join(root, ".git")));
|
|
1159
|
-
const recentArticles: { article: string; mtimeMs: number }[] = [];
|
|
1160
|
-
if (exists) {
|
|
1161
|
-
for (const article of stems) {
|
|
1162
|
-
try {
|
|
1163
|
-
const s = await stat(join(articlesDir(root), `${article}.md`));
|
|
1164
|
-
recentArticles.push({ article, mtimeMs: s.mtimeMs });
|
|
1165
|
-
} catch {
|
|
1166
|
-
// Ignore races/deleted files; status is best-effort.
|
|
1167
|
-
}
|
|
1168
|
-
}
|
|
1169
|
-
recentArticles.sort((a, b) => b.mtimeMs - a.mtimeMs);
|
|
1170
|
-
}
|
|
1171
|
-
return { root, exists, articleCount: stems.length, referenceCount: refs.length, isGitRepo, recentArticles: recentArticles.slice(0, 8) };
|
|
1172
|
-
}
|
|
1173
|
-
|
|
1174
|
-
/** Copy `*.md` articles from another wiki/vault's articles/ dir into this one
|
|
1175
|
-
* (skips files that already exist). Used by `memory init` to seed from an
|
|
1176
|
-
* existing vault. Returns the imported stems. */
|
|
1177
|
-
export async function importArticlesFrom(sourceArticlesDir: string, root = vaultRoot()): Promise<string[]> {
|
|
1178
|
-
let names: string[];
|
|
1179
|
-
try {
|
|
1180
|
-
names = await readdir(sourceArticlesDir);
|
|
1181
|
-
} catch {
|
|
1182
|
-
return [];
|
|
1183
|
-
}
|
|
1184
|
-
await mkdir(articlesDir(root), { recursive: true });
|
|
1185
|
-
const imported: string[] = [];
|
|
1186
|
-
for (const name of names) {
|
|
1187
|
-
if (!name.endsWith(".md") || name.startsWith(".")) continue;
|
|
1188
|
-
const dest = join(articlesDir(root), name);
|
|
1189
|
-
if (await pathExists(dest)) continue;
|
|
1190
|
-
await writeFile(dest, await readFile(join(sourceArticlesDir, name), "utf8"));
|
|
1191
|
-
imported.push(name.slice(0, -3));
|
|
1192
|
-
}
|
|
1193
|
-
return imported;
|
|
1194
|
-
}
|