create-egregore 0.21.6 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/egregore-launcher.js +58 -6
- package/assets/egregore-launcher.sh +15 -2
- package/lib/claude-runtime.js +2 -0
- package/lib/codex-runtime.js +2 -0
- package/lib/setup.js +20 -3
- package/package.json +2 -2
- package/runtime/claude/.claude/context/commit-format.md +7 -4
- package/runtime/claude/.claude/context/handoff-claiming.md +32 -0
- package/runtime/claude/.claude/context/notification-consent.md +12 -9
- package/runtime/claude/.claude/hooks/boundary-check.sh +14 -331
- package/runtime/claude/.claude/hooks/memory-read-guard.sh +9 -1
- package/runtime/claude/.claude/rules/voice-bedrock.md +6 -2
- package/runtime/claude/.claude/settings.json +1 -1
- package/runtime/claude/.claude/skills/commit/SKILL.md +5 -3
- package/runtime/claude/.claude/skills/dark-mode/SKILL.md +48 -369
- package/runtime/claude/.claude/skills/records/SKILL.md +158 -0
- package/runtime/claude/.claude/skills/reflect/SKILL.md +50 -21
- package/runtime/claude/.claude/skills/review-pr/SKILL.md +4 -2
- package/runtime/claude/.claude/skills/save/SKILL.md +9 -3
- package/runtime/claude/.claude/skills/summon/SKILL.md +138 -207
- package/runtime/claude/.claude/skills/tui-design/SKILL.md +1 -1
- package/runtime/claude/.claude/skills/update/SKILL.md +1 -1
- package/runtime/claude/.claude/skills/view/SKILL.md +52 -16
- package/runtime/claude/.claude/skills/view/compose-scaffold.html +19 -13
- package/runtime/claude/CLAUDE.md +59 -95
- package/runtime/claude/bin/agent.sh +231 -7
- package/runtime/claude/bin/api-call.sh +44 -23
- package/runtime/claude/bin/artifact-register.sh +7 -1
- package/runtime/claude/bin/artifact-writeback.sh +1 -1
- package/runtime/claude/bin/artifacts.sh +66 -265
- package/runtime/claude/bin/attendant.sh +4 -1
- package/runtime/claude/bin/boundary-guard.cjs +1848 -0
- package/runtime/claude/bin/boundary-hook.sh +41 -0
- package/runtime/claude/bin/boundary.sh +15 -155
- package/runtime/claude/bin/capture-run.sh +32 -2
- package/runtime/claude/bin/codex-render-spec.mjs +24 -46
- package/runtime/claude/bin/codex-session-start.sh +19 -0
- package/runtime/claude/bin/codex-skill-render.mjs +18 -0
- package/runtime/claude/bin/config-get.sh +10 -1
- package/runtime/claude/bin/connect-refresh.sh +16 -0
- package/runtime/claude/bin/corpus_map.py +5 -0
- package/runtime/claude/bin/data/capabilities/claude.json +1480 -0
- package/runtime/claude/bin/genui.sh +19 -0
- package/runtime/claude/bin/graph-batch.sh +1 -1
- package/runtime/claude/bin/graph-op.sh +14 -0
- package/runtime/claude/bin/graph.sh +1 -1
- package/runtime/claude/bin/handoff-pr-backfill.sh +1 -1
- package/runtime/claude/bin/handoff-run.sh +11 -6
- package/runtime/claude/bin/ingest-item.sh +1 -1
- package/runtime/claude/bin/ingest-surface.html +22 -4
- package/runtime/claude/bin/issue.sh +1 -1
- package/runtime/claude/bin/knowledge.sh +1 -1
- package/runtime/claude/bin/lib/card-delivery.sh +329 -0
- package/runtime/claude/bin/lib/config.sh +148 -3
- package/runtime/claude/bin/lib/context.sh +38 -67
- package/runtime/claude/bin/lib/genui-entry.py +36 -0
- package/runtime/claude/bin/lib/git-message.sh +25 -6
- package/runtime/claude/bin/lib/git-sync.sh +12 -4
- package/runtime/claude/bin/lib/greeting.sh +255 -28
- package/runtime/claude/bin/lib/handoff_meta.py +294 -0
- package/runtime/claude/bin/lib/health-footer.sh +83 -2
- package/runtime/claude/bin/lib/identity.sh +31 -10
- package/runtime/claude/bin/lib/instance-registry.sh +71 -0
- package/runtime/claude/bin/lib/local_ci_receipt.py +249 -0
- package/runtime/claude/bin/lib/metrics.sh +121 -8
- package/runtime/claude/bin/lib/notices.sh +36 -0
- package/runtime/claude/bin/lib/record_outcomes.py +729 -0
- package/runtime/claude/bin/lib/reflexes.py +267 -0
- package/runtime/claude/bin/lib/retrieval_scorecard.py +624 -0
- package/runtime/claude/bin/lib/session-record-prompt.txt +18 -0
- package/runtime/claude/bin/lib/session_record.py +3747 -0
- package/runtime/claude/bin/lib/session_trail.py +1983 -0
- package/runtime/claude/bin/lib/settings-drift.sh +8 -5
- package/runtime/claude/bin/lib/startup-restart.sh +41 -0
- package/runtime/claude/bin/lib/worktree-links.sh +92 -0
- package/runtime/claude/bin/memory-push.sh +6 -0
- package/runtime/claude/bin/merge-gate.sh +498 -0
- package/runtime/claude/bin/notify.sh +5 -2
- package/runtime/claude/bin/page-access.sh +103 -0
- package/runtime/claude/bin/person.sh +13 -2
- package/runtime/claude/bin/pi-render-spec.mjs +0 -1
- package/runtime/claude/bin/pi-spirit-discovery.mjs +34 -0
- package/runtime/claude/bin/post-update.sh +181 -0
- package/runtime/claude/bin/prime-render-spec.mjs +0 -1
- package/runtime/claude/bin/publish-artifact.sh +89 -42
- package/runtime/claude/bin/qa-inventory.sh +2 -2
- package/runtime/claude/bin/quest.sh +1 -1
- package/runtime/claude/bin/records.sh +11 -0
- package/runtime/claude/bin/reflexes/05-vitals.sh +9 -0
- package/runtime/claude/bin/reflexes/10-open-items.sh +12 -0
- package/runtime/claude/bin/reflexes/15-retrieval.sh +8 -0
- package/runtime/claude/bin/reflexes/20-nightly.sh +100 -0
- package/runtime/claude/bin/reflexes/20-sandbox.sh +85 -0
- package/runtime/claude/bin/reflexes/30-cost.py +116 -0
- package/runtime/claude/bin/reflexes/40-shells.py +51 -0
- package/runtime/claude/bin/reflexes/README.md +61 -0
- package/runtime/claude/bin/reflexes.sh +7 -0
- package/runtime/claude/bin/runtime-identity.sh +140 -42
- package/runtime/claude/bin/screenshot.sh +77 -0
- package/runtime/claude/bin/scroll.sh +1 -1
- package/runtime/claude/bin/search.sh +31 -5
- package/runtime/claude/bin/session-autosave.sh +6 -3
- package/runtime/claude/bin/session-id.sh +147 -0
- package/runtime/claude/bin/session-log.sh +19 -0
- package/runtime/claude/bin/session-record.sh +26 -0
- package/runtime/claude/bin/session-start.sh +121 -151
- package/runtime/claude/bin/settings.sh +13 -8
- package/runtime/claude/bin/spirit-discovery-claude.sh +4 -0
- package/runtime/claude/bin/spirit-discovery-codex.sh +4 -0
- package/runtime/claude/bin/spirit-discovery-pi.sh +4 -0
- package/runtime/claude/bin/spirit-discovery-prime.sh +4 -0
- package/runtime/claude/bin/spirit-discovery.sh +4 -0
- package/runtime/claude/bin/startup-check.sh +1 -1
- package/runtime/claude/bin/telemetry.sh +11 -4
- package/runtime/claude/bin/test-baseline.sh +33 -5
- package/runtime/claude/bin/todo.sh +1 -1
- package/runtime/claude/bin/worktree-create.sh +15 -45
- package/runtime/claude/bin/worktree.sh +6 -0
- package/runtime/claude/docs/spirit-discovery.md +96 -0
- package/runtime/claude/egregore_runtime/__init__.py +18 -12
- package/runtime/claude/egregore_runtime/access.py +608 -0
- package/runtime/claude/egregore_runtime/access_cli.py +190 -0
- package/runtime/claude/egregore_runtime/adapters/invitation.py +5 -3
- package/runtime/claude/egregore_runtime/adapters/qmd.py +521 -47
- package/runtime/claude/egregore_runtime/admin_entry.py +56 -0
- package/runtime/claude/egregore_runtime/artifacts.py +222 -16
- package/runtime/claude/egregore_runtime/contracts.py +27 -0
- package/runtime/claude/egregore_runtime/eligibility.py +5 -12
- package/runtime/claude/egregore_runtime/episode_binding.py +16 -5
- package/runtime/claude/egregore_runtime/github_issues.py +2 -2
- package/runtime/claude/egregore_runtime/handoff_lookup.py +9 -20
- package/runtime/claude/egregore_runtime/harness_cli.py +674 -149
- package/runtime/claude/egregore_runtime/held.py +2436 -0
- package/runtime/claude/egregore_runtime/hosted_api.py +137 -0
- package/runtime/claude/egregore_runtime/identity.py +196 -65
- package/runtime/claude/egregore_runtime/identity_transfer.py +6 -2
- package/runtime/claude/egregore_runtime/ingest_cli.py +2 -2
- package/runtime/claude/egregore_runtime/ingest_index_cli.py +4 -16
- package/runtime/claude/egregore_runtime/installation.py +119 -0
- package/runtime/claude/egregore_runtime/investigation.py +64 -41
- package/runtime/claude/egregore_runtime/investigation_guidance.py +16 -11
- package/runtime/claude/egregore_runtime/issue_cli.py +51 -32
- package/runtime/claude/egregore_runtime/issue_reporting.py +4 -3
- package/runtime/claude/egregore_runtime/issue_triage_service.py +2 -2
- package/runtime/claude/egregore_runtime/issues.py +12 -6
- package/runtime/claude/egregore_runtime/knowledge.py +331 -5
- package/runtime/claude/egregore_runtime/knowledge_cli.py +57 -17
- package/runtime/claude/egregore_runtime/lifecycle.py +88 -22
- package/runtime/claude/egregore_runtime/lifecycle_cli.py +21 -13
- package/runtime/claude/egregore_runtime/line_ids.py +10 -0
- package/runtime/claude/egregore_runtime/memory_push.py +349 -0
- package/runtime/claude/egregore_runtime/observe.py +64 -25
- package/runtime/claude/egregore_runtime/observers.py +19 -0
- package/runtime/claude/egregore_runtime/optional_modules.py +13 -0
- package/runtime/claude/egregore_runtime/outputs_cli.py +321 -0
- package/runtime/claude/egregore_runtime/policy.py +651 -24
- package/runtime/claude/egregore_runtime/private_result.py +2 -1
- package/runtime/claude/egregore_runtime/projection.py +946 -0
- package/runtime/claude/egregore_runtime/quest_cli.py +20 -23
- package/runtime/claude/egregore_runtime/question_cli.py +15 -11
- package/runtime/claude/egregore_runtime/questions.py +10 -10
- package/runtime/claude/egregore_runtime/quests.py +10 -10
- package/runtime/claude/egregore_runtime/record_publish.py +548 -0
- package/runtime/claude/egregore_runtime/record_subject.py +564 -0
- package/runtime/claude/egregore_runtime/research_ingest.py +2 -4
- package/runtime/claude/egregore_runtime/runtime.py +169 -60
- package/runtime/claude/egregore_runtime/sandbox.py +194 -0
- package/runtime/claude/egregore_runtime/scroll_cli.py +22 -32
- package/runtime/claude/egregore_runtime/scrolls.py +23 -15
- package/runtime/claude/egregore_runtime/shadow.py +262 -0
- package/runtime/claude/egregore_runtime/spirit_discovery.py +487 -0
- package/runtime/claude/egregore_runtime/spirit_discovery_cli.py +25 -0
- package/runtime/claude/egregore_runtime/spirit_discovery_transport.py +18 -0
- package/runtime/claude/egregore_runtime/status.py +101 -44
- package/runtime/claude/egregore_runtime/status_cli.py +13 -5
- package/runtime/claude/egregore_runtime/sync.py +39 -21
- package/runtime/claude/egregore_runtime/taste.py +934 -0
- package/runtime/claude/egregore_runtime/taste_cli.py +242 -0
- package/runtime/claude/egregore_runtime/taste_rules.py +91 -0
- package/runtime/claude/egregore_runtime/telemetry.py +26 -3
- package/runtime/claude/egregore_runtime/todo_cli.py +15 -12
- package/runtime/claude/egregore_runtime/todos.py +3 -2
- package/runtime/claude/egregore_runtime/upgrade.py +85 -58
- package/runtime/claude/egregore_runtime/writeback.py +330 -24
- package/runtime/claude/manifest.json +65 -5
- package/runtime/codex/.claude/context/handoff-claiming.md +32 -0
- package/runtime/codex/.claude/context/notification-consent.md +12 -9
- package/runtime/codex/.claude/skills/view/SKILL.md +438 -0
- package/runtime/codex/.codex/hooks.json +11 -0
- package/runtime/codex/.codex/skills/dark-mode/SKILL.md +1 -1
- package/runtime/codex/.codex/skills/records/SKILL.md +29 -0
- package/runtime/codex/.codex/skills/summon/SKILL.md +1 -1
- package/runtime/codex/.codex/skills/view/assets/compose-scaffold.html +19 -13
- package/runtime/codex/.codex/spec-manifest.json +13 -19
- package/runtime/codex/AGENTS.md +54 -105
- package/runtime/codex/bin/agent.sh +231 -7
- package/runtime/codex/bin/api-call.sh +44 -23
- package/runtime/codex/bin/artifact-register.sh +7 -1
- package/runtime/codex/bin/artifact-writeback.sh +1 -1
- package/runtime/codex/bin/artifacts.sh +66 -265
- package/runtime/codex/bin/attendant.sh +4 -1
- package/runtime/codex/bin/boundary-guard.cjs +1848 -0
- package/runtime/codex/bin/boundary-hook.sh +41 -0
- package/runtime/codex/bin/capture-run.sh +32 -2
- package/runtime/codex/bin/codex-render-spec.mjs +24 -46
- package/runtime/codex/bin/codex-session-start.sh +19 -0
- package/runtime/codex/bin/codex-skill-render.mjs +18 -0
- package/runtime/codex/bin/config-get.sh +10 -1
- package/runtime/codex/bin/connect-refresh.sh +16 -0
- package/runtime/codex/bin/corpus_map.py +5 -0
- package/runtime/codex/bin/data/capabilities/codex.json +1458 -0
- package/runtime/codex/bin/genui.sh +19 -0
- package/runtime/codex/bin/graph-batch.sh +1 -1
- package/runtime/codex/bin/graph-op.sh +14 -0
- package/runtime/codex/bin/graph.sh +1 -1
- package/runtime/codex/bin/handoff-pr-backfill.sh +1 -1
- package/runtime/codex/bin/handoff-run.sh +11 -6
- package/runtime/codex/bin/ingest-item.sh +1 -1
- package/runtime/codex/bin/ingest-surface.html +22 -4
- package/runtime/codex/bin/issue.sh +1 -1
- package/runtime/codex/bin/knowledge.sh +1 -1
- package/runtime/codex/bin/lib/card-delivery.sh +329 -0
- package/runtime/codex/bin/lib/config.sh +148 -3
- package/runtime/codex/bin/lib/context.sh +38 -67
- package/runtime/codex/bin/lib/genui-entry.py +36 -0
- package/runtime/codex/bin/lib/git-message.sh +25 -6
- package/runtime/codex/bin/lib/git-sync.sh +12 -4
- package/runtime/codex/bin/lib/greeting.sh +255 -28
- package/runtime/codex/bin/lib/handoff_meta.py +294 -0
- package/runtime/codex/bin/lib/health-footer.sh +83 -2
- package/runtime/codex/bin/lib/identity.sh +31 -10
- package/runtime/codex/bin/lib/instance-registry.sh +71 -0
- package/runtime/codex/bin/lib/local_ci_receipt.py +249 -0
- package/runtime/codex/bin/lib/metrics.sh +121 -8
- package/runtime/codex/bin/lib/notices.sh +36 -0
- package/runtime/codex/bin/lib/record_outcomes.py +729 -0
- package/runtime/codex/bin/lib/reflexes.py +267 -0
- package/runtime/codex/bin/lib/retrieval_scorecard.py +624 -0
- package/runtime/codex/bin/lib/session-record-prompt.txt +18 -0
- package/runtime/codex/bin/lib/session_record.py +3747 -0
- package/runtime/codex/bin/lib/session_trail.py +1983 -0
- package/runtime/codex/bin/lib/settings-drift.sh +8 -5
- package/runtime/codex/bin/lib/startup-restart.sh +41 -0
- package/runtime/codex/bin/lib/worktree-links.sh +92 -0
- package/runtime/codex/bin/memory-push.sh +6 -0
- package/runtime/codex/bin/merge-gate.sh +498 -0
- package/runtime/codex/bin/notify.sh +5 -2
- package/runtime/codex/bin/page-access.sh +103 -0
- package/runtime/codex/bin/person.sh +13 -2
- package/runtime/codex/bin/pi-render-spec.mjs +0 -1
- package/runtime/codex/bin/pi-spirit-discovery.mjs +34 -0
- package/runtime/codex/bin/post-update.sh +181 -0
- package/runtime/codex/bin/prime-render-spec.mjs +0 -1
- package/runtime/codex/bin/publish-artifact.sh +89 -42
- package/runtime/codex/bin/qa-inventory.sh +2 -2
- package/runtime/codex/bin/quest.sh +1 -1
- package/runtime/codex/bin/records.sh +11 -0
- package/runtime/codex/bin/reflexes/05-vitals.sh +9 -0
- package/runtime/codex/bin/reflexes/10-open-items.sh +12 -0
- package/runtime/codex/bin/reflexes/15-retrieval.sh +8 -0
- package/runtime/codex/bin/reflexes/20-nightly.sh +100 -0
- package/runtime/codex/bin/reflexes/20-sandbox.sh +85 -0
- package/runtime/codex/bin/reflexes/30-cost.py +116 -0
- package/runtime/codex/bin/reflexes/40-shells.py +51 -0
- package/runtime/codex/bin/reflexes/README.md +61 -0
- package/runtime/codex/bin/reflexes.sh +7 -0
- package/runtime/codex/bin/runtime-identity.sh +140 -42
- package/runtime/codex/bin/screenshot.sh +77 -0
- package/runtime/codex/bin/scroll.sh +1 -1
- package/runtime/codex/bin/search.sh +31 -5
- package/runtime/codex/bin/session-autosave.sh +6 -3
- package/runtime/codex/bin/session-id.sh +147 -0
- package/runtime/codex/bin/session-log.sh +19 -0
- package/runtime/codex/bin/session-record.sh +26 -0
- package/runtime/codex/bin/settings.sh +13 -8
- package/runtime/codex/bin/spirit-discovery-claude.sh +4 -0
- package/runtime/codex/bin/spirit-discovery-codex.sh +4 -0
- package/runtime/codex/bin/spirit-discovery-pi.sh +4 -0
- package/runtime/codex/bin/spirit-discovery-prime.sh +4 -0
- package/runtime/codex/bin/spirit-discovery.sh +4 -0
- package/runtime/codex/bin/startup-check.sh +1 -1
- package/runtime/codex/bin/telemetry.sh +11 -4
- package/runtime/codex/bin/test-baseline.sh +33 -5
- package/runtime/codex/bin/todo.sh +1 -1
- package/runtime/codex/bin/worktree-create.sh +15 -45
- package/runtime/codex/bin/worktree.sh +6 -0
- package/runtime/codex/docs/spirit-discovery.md +96 -0
- package/runtime/codex/egregore_runtime/__init__.py +18 -12
- package/runtime/codex/egregore_runtime/access.py +608 -0
- package/runtime/codex/egregore_runtime/access_cli.py +190 -0
- package/runtime/codex/egregore_runtime/adapters/invitation.py +5 -3
- package/runtime/codex/egregore_runtime/adapters/qmd.py +521 -47
- package/runtime/codex/egregore_runtime/admin_entry.py +56 -0
- package/runtime/codex/egregore_runtime/artifacts.py +222 -16
- package/runtime/codex/egregore_runtime/contracts.py +27 -0
- package/runtime/codex/egregore_runtime/eligibility.py +5 -12
- package/runtime/codex/egregore_runtime/episode_binding.py +16 -5
- package/runtime/codex/egregore_runtime/github_issues.py +2 -2
- package/runtime/codex/egregore_runtime/handoff_lookup.py +9 -20
- package/runtime/codex/egregore_runtime/harness_cli.py +674 -149
- package/runtime/codex/egregore_runtime/held.py +2436 -0
- package/runtime/codex/egregore_runtime/hosted_api.py +137 -0
- package/runtime/codex/egregore_runtime/identity.py +196 -65
- package/runtime/codex/egregore_runtime/identity_transfer.py +6 -2
- package/runtime/codex/egregore_runtime/ingest_cli.py +2 -2
- package/runtime/codex/egregore_runtime/ingest_index_cli.py +4 -16
- package/runtime/codex/egregore_runtime/installation.py +119 -0
- package/runtime/codex/egregore_runtime/investigation.py +64 -41
- package/runtime/codex/egregore_runtime/investigation_guidance.py +16 -11
- package/runtime/codex/egregore_runtime/issue_cli.py +51 -32
- package/runtime/codex/egregore_runtime/issue_reporting.py +4 -3
- package/runtime/codex/egregore_runtime/issue_triage_service.py +2 -2
- package/runtime/codex/egregore_runtime/issues.py +12 -6
- package/runtime/codex/egregore_runtime/knowledge.py +331 -5
- package/runtime/codex/egregore_runtime/knowledge_cli.py +57 -17
- package/runtime/codex/egregore_runtime/lifecycle.py +88 -22
- package/runtime/codex/egregore_runtime/lifecycle_cli.py +21 -13
- package/runtime/codex/egregore_runtime/line_ids.py +10 -0
- package/runtime/codex/egregore_runtime/memory_push.py +349 -0
- package/runtime/codex/egregore_runtime/observe.py +64 -25
- package/runtime/codex/egregore_runtime/observers.py +19 -0
- package/runtime/codex/egregore_runtime/optional_modules.py +13 -0
- package/runtime/codex/egregore_runtime/outputs_cli.py +321 -0
- package/runtime/codex/egregore_runtime/policy.py +651 -24
- package/runtime/codex/egregore_runtime/private_result.py +2 -1
- package/runtime/codex/egregore_runtime/projection.py +946 -0
- package/runtime/codex/egregore_runtime/quest_cli.py +20 -23
- package/runtime/codex/egregore_runtime/question_cli.py +15 -11
- package/runtime/codex/egregore_runtime/questions.py +10 -10
- package/runtime/codex/egregore_runtime/quests.py +10 -10
- package/runtime/codex/egregore_runtime/record_publish.py +548 -0
- package/runtime/codex/egregore_runtime/record_subject.py +564 -0
- package/runtime/codex/egregore_runtime/research_ingest.py +2 -4
- package/runtime/codex/egregore_runtime/runtime.py +169 -60
- package/runtime/codex/egregore_runtime/sandbox.py +194 -0
- package/runtime/codex/egregore_runtime/scroll_cli.py +22 -32
- package/runtime/codex/egregore_runtime/scrolls.py +23 -15
- package/runtime/codex/egregore_runtime/shadow.py +262 -0
- package/runtime/codex/egregore_runtime/spirit_discovery.py +487 -0
- package/runtime/codex/egregore_runtime/spirit_discovery_cli.py +25 -0
- package/runtime/codex/egregore_runtime/spirit_discovery_transport.py +18 -0
- package/runtime/codex/egregore_runtime/status.py +101 -44
- package/runtime/codex/egregore_runtime/status_cli.py +13 -5
- package/runtime/codex/egregore_runtime/sync.py +39 -21
- package/runtime/codex/egregore_runtime/taste.py +934 -0
- package/runtime/codex/egregore_runtime/taste_cli.py +242 -0
- package/runtime/codex/egregore_runtime/taste_rules.py +91 -0
- package/runtime/codex/egregore_runtime/telemetry.py +26 -3
- package/runtime/codex/egregore_runtime/todo_cli.py +15 -12
- package/runtime/codex/egregore_runtime/todos.py +3 -2
- package/runtime/codex/egregore_runtime/upgrade.py +85 -58
- package/runtime/codex/egregore_runtime/writeback.py +330 -24
- package/runtime/codex/manifest.json +66 -5
- package/runtime/pi/.claude/context/handoff-claiming.md +32 -0
- package/runtime/pi/.claude/context/notification-consent.md +12 -9
- package/runtime/pi/.claude/skills/view/SKILL.md +438 -0
- package/runtime/pi/.codex/hooks.json +11 -0
- package/runtime/pi/.codex/skills/dark-mode/SKILL.md +1 -1
- package/runtime/pi/.codex/skills/records/SKILL.md +29 -0
- package/runtime/pi/.codex/skills/summon/SKILL.md +1 -1
- package/runtime/pi/.codex/skills/view/assets/compose-scaffold.html +19 -13
- package/runtime/pi/.codex/spec-manifest.json +13 -19
- package/runtime/pi/.pi/APPEND_SYSTEM.md +53 -99
- package/runtime/pi/.pi/extensions/egregore.ts +13 -1
- package/runtime/pi/.pi/spec-manifest.json +2 -2
- package/runtime/pi/AGENTS.md +54 -105
- package/runtime/pi/bin/agent.sh +231 -7
- package/runtime/pi/bin/api-call.sh +44 -23
- package/runtime/pi/bin/artifact-register.sh +7 -1
- package/runtime/pi/bin/artifact-writeback.sh +1 -1
- package/runtime/pi/bin/artifacts.sh +66 -265
- package/runtime/pi/bin/attendant.sh +4 -1
- package/runtime/pi/bin/boundary-guard.cjs +1848 -0
- package/runtime/pi/bin/boundary-hook.sh +41 -0
- package/runtime/pi/bin/capture-run.sh +32 -2
- package/runtime/pi/bin/codex-render-spec.mjs +24 -46
- package/runtime/pi/bin/codex-session-start.sh +19 -0
- package/runtime/pi/bin/codex-skill-render.mjs +18 -0
- package/runtime/pi/bin/config-get.sh +10 -1
- package/runtime/pi/bin/connect-refresh.sh +16 -0
- package/runtime/pi/bin/corpus_map.py +5 -0
- package/runtime/pi/bin/data/capabilities/pi.json +1458 -0
- package/runtime/pi/bin/genui.sh +19 -0
- package/runtime/pi/bin/graph-batch.sh +1 -1
- package/runtime/pi/bin/graph-op.sh +14 -0
- package/runtime/pi/bin/graph.sh +1 -1
- package/runtime/pi/bin/handoff-pr-backfill.sh +1 -1
- package/runtime/pi/bin/handoff-run.sh +11 -6
- package/runtime/pi/bin/ingest-item.sh +1 -1
- package/runtime/pi/bin/ingest-surface.html +22 -4
- package/runtime/pi/bin/issue.sh +1 -1
- package/runtime/pi/bin/knowledge.sh +1 -1
- package/runtime/pi/bin/lib/card-delivery.sh +329 -0
- package/runtime/pi/bin/lib/config.sh +148 -3
- package/runtime/pi/bin/lib/context.sh +38 -67
- package/runtime/pi/bin/lib/genui-entry.py +36 -0
- package/runtime/pi/bin/lib/git-message.sh +25 -6
- package/runtime/pi/bin/lib/git-sync.sh +12 -4
- package/runtime/pi/bin/lib/greeting.sh +255 -28
- package/runtime/pi/bin/lib/handoff_meta.py +294 -0
- package/runtime/pi/bin/lib/health-footer.sh +83 -2
- package/runtime/pi/bin/lib/identity.sh +31 -10
- package/runtime/pi/bin/lib/instance-registry.sh +71 -0
- package/runtime/pi/bin/lib/local_ci_receipt.py +249 -0
- package/runtime/pi/bin/lib/metrics.sh +121 -8
- package/runtime/pi/bin/lib/notices.sh +36 -0
- package/runtime/pi/bin/lib/record_outcomes.py +729 -0
- package/runtime/pi/bin/lib/reflexes.py +267 -0
- package/runtime/pi/bin/lib/retrieval_scorecard.py +624 -0
- package/runtime/pi/bin/lib/session-record-prompt.txt +18 -0
- package/runtime/pi/bin/lib/session_record.py +3747 -0
- package/runtime/pi/bin/lib/session_trail.py +1983 -0
- package/runtime/pi/bin/lib/settings-drift.sh +8 -5
- package/runtime/pi/bin/lib/startup-restart.sh +41 -0
- package/runtime/pi/bin/lib/worktree-links.sh +92 -0
- package/runtime/pi/bin/memory-push.sh +6 -0
- package/runtime/pi/bin/merge-gate.sh +498 -0
- package/runtime/pi/bin/notify.sh +5 -2
- package/runtime/pi/bin/page-access.sh +103 -0
- package/runtime/pi/bin/person.sh +13 -2
- package/runtime/pi/bin/pi-branch-policy.mjs +37 -0
- package/runtime/pi/bin/pi-observe-context.mjs +97 -4
- package/runtime/pi/bin/pi-render-spec.mjs +0 -1
- package/runtime/pi/bin/pi-session-start.sh +4 -1
- package/runtime/pi/bin/pi-spirit-discovery.mjs +34 -0
- package/runtime/pi/bin/post-update.sh +181 -0
- package/runtime/pi/bin/prime-render-spec.mjs +0 -1
- package/runtime/pi/bin/publish-artifact.sh +89 -42
- package/runtime/pi/bin/qa-inventory.sh +2 -2
- package/runtime/pi/bin/quest.sh +1 -1
- package/runtime/pi/bin/records.sh +11 -0
- package/runtime/pi/bin/reflexes/05-vitals.sh +9 -0
- package/runtime/pi/bin/reflexes/10-open-items.sh +12 -0
- package/runtime/pi/bin/reflexes/15-retrieval.sh +8 -0
- package/runtime/pi/bin/reflexes/20-nightly.sh +100 -0
- package/runtime/pi/bin/reflexes/20-sandbox.sh +85 -0
- package/runtime/pi/bin/reflexes/30-cost.py +116 -0
- package/runtime/pi/bin/reflexes/40-shells.py +51 -0
- package/runtime/pi/bin/reflexes/README.md +61 -0
- package/runtime/pi/bin/reflexes.sh +7 -0
- package/runtime/pi/bin/runtime-identity.sh +140 -42
- package/runtime/pi/bin/screenshot.sh +77 -0
- package/runtime/pi/bin/scroll.sh +1 -1
- package/runtime/pi/bin/search.sh +31 -5
- package/runtime/pi/bin/session-autosave.sh +6 -3
- package/runtime/pi/bin/session-id.sh +147 -0
- package/runtime/pi/bin/session-log.sh +19 -0
- package/runtime/pi/bin/session-record.sh +26 -0
- package/runtime/pi/bin/settings.sh +13 -8
- package/runtime/pi/bin/spirit-discovery-claude.sh +4 -0
- package/runtime/pi/bin/spirit-discovery-codex.sh +4 -0
- package/runtime/pi/bin/spirit-discovery-pi.sh +4 -0
- package/runtime/pi/bin/spirit-discovery-prime.sh +4 -0
- package/runtime/pi/bin/spirit-discovery.sh +4 -0
- package/runtime/pi/bin/startup-check.sh +1 -1
- package/runtime/pi/bin/telemetry.sh +11 -4
- package/runtime/pi/bin/test-baseline.sh +33 -5
- package/runtime/pi/bin/todo.sh +1 -1
- package/runtime/pi/bin/worktree-create.sh +15 -45
- package/runtime/pi/bin/worktree.sh +6 -0
- package/runtime/pi/docs/spirit-discovery.md +96 -0
- package/runtime/pi/egregore_runtime/__init__.py +18 -12
- package/runtime/pi/egregore_runtime/access.py +608 -0
- package/runtime/pi/egregore_runtime/access_cli.py +190 -0
- package/runtime/pi/egregore_runtime/adapters/invitation.py +5 -3
- package/runtime/pi/egregore_runtime/adapters/qmd.py +521 -47
- package/runtime/pi/egregore_runtime/admin_entry.py +56 -0
- package/runtime/pi/egregore_runtime/artifacts.py +222 -16
- package/runtime/pi/egregore_runtime/contracts.py +27 -0
- package/runtime/pi/egregore_runtime/eligibility.py +5 -12
- package/runtime/pi/egregore_runtime/episode_binding.py +16 -5
- package/runtime/pi/egregore_runtime/github_issues.py +2 -2
- package/runtime/pi/egregore_runtime/handoff_lookup.py +9 -20
- package/runtime/pi/egregore_runtime/harness_cli.py +674 -149
- package/runtime/pi/egregore_runtime/held.py +2436 -0
- package/runtime/pi/egregore_runtime/hosted_api.py +137 -0
- package/runtime/pi/egregore_runtime/identity.py +196 -65
- package/runtime/pi/egregore_runtime/identity_transfer.py +6 -2
- package/runtime/pi/egregore_runtime/ingest_cli.py +2 -2
- package/runtime/pi/egregore_runtime/ingest_index_cli.py +4 -16
- package/runtime/pi/egregore_runtime/installation.py +119 -0
- package/runtime/pi/egregore_runtime/investigation.py +64 -41
- package/runtime/pi/egregore_runtime/investigation_guidance.py +16 -11
- package/runtime/pi/egregore_runtime/issue_cli.py +51 -32
- package/runtime/pi/egregore_runtime/issue_reporting.py +4 -3
- package/runtime/pi/egregore_runtime/issue_triage_service.py +2 -2
- package/runtime/pi/egregore_runtime/issues.py +12 -6
- package/runtime/pi/egregore_runtime/knowledge.py +331 -5
- package/runtime/pi/egregore_runtime/knowledge_cli.py +57 -17
- package/runtime/pi/egregore_runtime/lifecycle.py +88 -22
- package/runtime/pi/egregore_runtime/lifecycle_cli.py +21 -13
- package/runtime/pi/egregore_runtime/line_ids.py +10 -0
- package/runtime/pi/egregore_runtime/memory_push.py +349 -0
- package/runtime/pi/egregore_runtime/observe.py +64 -25
- package/runtime/pi/egregore_runtime/observers.py +19 -0
- package/runtime/pi/egregore_runtime/optional_modules.py +13 -0
- package/runtime/pi/egregore_runtime/outputs_cli.py +321 -0
- package/runtime/pi/egregore_runtime/policy.py +651 -24
- package/runtime/pi/egregore_runtime/private_result.py +2 -1
- package/runtime/pi/egregore_runtime/projection.py +946 -0
- package/runtime/pi/egregore_runtime/quest_cli.py +20 -23
- package/runtime/pi/egregore_runtime/question_cli.py +15 -11
- package/runtime/pi/egregore_runtime/questions.py +10 -10
- package/runtime/pi/egregore_runtime/quests.py +10 -10
- package/runtime/pi/egregore_runtime/record_publish.py +548 -0
- package/runtime/pi/egregore_runtime/record_subject.py +564 -0
- package/runtime/pi/egregore_runtime/research_ingest.py +2 -4
- package/runtime/pi/egregore_runtime/runtime.py +169 -60
- package/runtime/pi/egregore_runtime/sandbox.py +194 -0
- package/runtime/pi/egregore_runtime/scroll_cli.py +22 -32
- package/runtime/pi/egregore_runtime/scrolls.py +23 -15
- package/runtime/pi/egregore_runtime/shadow.py +262 -0
- package/runtime/pi/egregore_runtime/spirit_discovery.py +487 -0
- package/runtime/pi/egregore_runtime/spirit_discovery_cli.py +25 -0
- package/runtime/pi/egregore_runtime/spirit_discovery_transport.py +18 -0
- package/runtime/pi/egregore_runtime/status.py +101 -44
- package/runtime/pi/egregore_runtime/status_cli.py +13 -5
- package/runtime/pi/egregore_runtime/sync.py +39 -21
- package/runtime/pi/egregore_runtime/taste.py +934 -0
- package/runtime/pi/egregore_runtime/taste_cli.py +242 -0
- package/runtime/pi/egregore_runtime/taste_rules.py +91 -0
- package/runtime/pi/egregore_runtime/telemetry.py +26 -3
- package/runtime/pi/egregore_runtime/todo_cli.py +15 -12
- package/runtime/pi/egregore_runtime/todos.py +3 -2
- package/runtime/pi/egregore_runtime/upgrade.py +85 -58
- package/runtime/pi/egregore_runtime/writeback.py +330 -24
- package/runtime/pi/manifest.json +66 -5
- package/runtime/prime/.claude/context/handoff-claiming.md +32 -0
- package/runtime/prime/.claude/context/notification-consent.md +12 -9
- package/runtime/prime/.claude/skills/view/SKILL.md +438 -0
- package/runtime/prime/.codex/hooks.json +11 -0
- package/runtime/prime/.codex/skills/dark-mode/SKILL.md +1 -1
- package/runtime/prime/.codex/skills/records/SKILL.md +29 -0
- package/runtime/prime/.codex/skills/summon/SKILL.md +1 -1
- package/runtime/prime/.codex/skills/view/assets/compose-scaffold.html +19 -13
- package/runtime/prime/.codex/spec-manifest.json +13 -19
- package/runtime/prime/.prime/agent/APPEND_SYSTEM.md +53 -99
- package/runtime/prime/.prime/agent/extensions/egregore.ts +14 -1
- package/runtime/prime/.prime/agent/skills/egregore/SKILL.md +14 -6
- package/runtime/prime/.prime/agent/skills/egregore/src/egregore/__init__.py +66 -1
- package/runtime/prime/.prime/agent/spec-manifest.json +2 -2
- package/runtime/prime/AGENTS.md +54 -105
- package/runtime/prime/bin/agent.sh +231 -7
- package/runtime/prime/bin/api-call.sh +44 -23
- package/runtime/prime/bin/artifact-register.sh +7 -1
- package/runtime/prime/bin/artifact-writeback.sh +1 -1
- package/runtime/prime/bin/artifacts.sh +66 -265
- package/runtime/prime/bin/attendant.sh +4 -1
- package/runtime/prime/bin/boundary-guard.cjs +1848 -0
- package/runtime/prime/bin/boundary-hook.sh +41 -0
- package/runtime/prime/bin/capture-run.sh +32 -2
- package/runtime/prime/bin/codex-render-spec.mjs +24 -46
- package/runtime/prime/bin/codex-session-start.sh +19 -0
- package/runtime/prime/bin/codex-skill-render.mjs +18 -0
- package/runtime/prime/bin/config-get.sh +10 -1
- package/runtime/prime/bin/connect-refresh.sh +16 -0
- package/runtime/prime/bin/corpus_map.py +5 -0
- package/runtime/prime/bin/data/capabilities/prime.json +1458 -0
- package/runtime/prime/bin/genui.sh +19 -0
- package/runtime/prime/bin/graph-batch.sh +1 -1
- package/runtime/prime/bin/graph-op.sh +14 -0
- package/runtime/prime/bin/graph.sh +1 -1
- package/runtime/prime/bin/handoff-pr-backfill.sh +1 -1
- package/runtime/prime/bin/handoff-run.sh +11 -6
- package/runtime/prime/bin/ingest-item.sh +1 -1
- package/runtime/prime/bin/ingest-surface.html +22 -4
- package/runtime/prime/bin/issue.sh +1 -1
- package/runtime/prime/bin/knowledge.sh +1 -1
- package/runtime/prime/bin/lib/card-delivery.sh +329 -0
- package/runtime/prime/bin/lib/config.sh +148 -3
- package/runtime/prime/bin/lib/context.sh +38 -67
- package/runtime/prime/bin/lib/genui-entry.py +36 -0
- package/runtime/prime/bin/lib/git-message.sh +25 -6
- package/runtime/prime/bin/lib/git-sync.sh +12 -4
- package/runtime/prime/bin/lib/greeting.sh +255 -28
- package/runtime/prime/bin/lib/handoff_meta.py +294 -0
- package/runtime/prime/bin/lib/health-footer.sh +83 -2
- package/runtime/prime/bin/lib/identity.sh +31 -10
- package/runtime/prime/bin/lib/instance-registry.sh +71 -0
- package/runtime/prime/bin/lib/local_ci_receipt.py +249 -0
- package/runtime/prime/bin/lib/metrics.sh +121 -8
- package/runtime/prime/bin/lib/notices.sh +36 -0
- package/runtime/prime/bin/lib/record_outcomes.py +729 -0
- package/runtime/prime/bin/lib/reflexes.py +267 -0
- package/runtime/prime/bin/lib/retrieval_scorecard.py +624 -0
- package/runtime/prime/bin/lib/session-record-prompt.txt +18 -0
- package/runtime/prime/bin/lib/session_record.py +3747 -0
- package/runtime/prime/bin/lib/session_trail.py +1983 -0
- package/runtime/prime/bin/lib/settings-drift.sh +8 -5
- package/runtime/prime/bin/lib/startup-restart.sh +41 -0
- package/runtime/prime/bin/lib/worktree-links.sh +92 -0
- package/runtime/prime/bin/memory-push.sh +6 -0
- package/runtime/prime/bin/merge-gate.sh +498 -0
- package/runtime/prime/bin/notify.sh +5 -2
- package/runtime/prime/bin/page-access.sh +103 -0
- package/runtime/prime/bin/person.sh +13 -2
- package/runtime/prime/bin/pi-branch-policy.mjs +37 -0
- package/runtime/prime/bin/pi-observe-context.mjs +97 -4
- package/runtime/prime/bin/pi-render-spec.mjs +0 -1
- package/runtime/prime/bin/pi-spirit-discovery.mjs +34 -0
- package/runtime/prime/bin/post-update.sh +181 -0
- package/runtime/prime/bin/prime-render-spec.mjs +0 -1
- package/runtime/prime/bin/prime-session-start.sh +3 -1
- package/runtime/prime/bin/publish-artifact.sh +89 -42
- package/runtime/prime/bin/qa-inventory.sh +2 -2
- package/runtime/prime/bin/quest.sh +1 -1
- package/runtime/prime/bin/records.sh +11 -0
- package/runtime/prime/bin/reflexes/05-vitals.sh +9 -0
- package/runtime/prime/bin/reflexes/10-open-items.sh +12 -0
- package/runtime/prime/bin/reflexes/15-retrieval.sh +8 -0
- package/runtime/prime/bin/reflexes/20-nightly.sh +100 -0
- package/runtime/prime/bin/reflexes/20-sandbox.sh +85 -0
- package/runtime/prime/bin/reflexes/30-cost.py +116 -0
- package/runtime/prime/bin/reflexes/40-shells.py +51 -0
- package/runtime/prime/bin/reflexes/README.md +61 -0
- package/runtime/prime/bin/reflexes.sh +7 -0
- package/runtime/prime/bin/runtime-identity.sh +140 -42
- package/runtime/prime/bin/screenshot.sh +77 -0
- package/runtime/prime/bin/scroll.sh +1 -1
- package/runtime/prime/bin/search.sh +31 -5
- package/runtime/prime/bin/session-autosave.sh +6 -3
- package/runtime/prime/bin/session-id.sh +147 -0
- package/runtime/prime/bin/session-log.sh +19 -0
- package/runtime/prime/bin/session-record.sh +26 -0
- package/runtime/prime/bin/settings.sh +13 -8
- package/runtime/prime/bin/spirit-discovery-claude.sh +4 -0
- package/runtime/prime/bin/spirit-discovery-codex.sh +4 -0
- package/runtime/prime/bin/spirit-discovery-pi.sh +4 -0
- package/runtime/prime/bin/spirit-discovery-prime.sh +4 -0
- package/runtime/prime/bin/spirit-discovery.sh +4 -0
- package/runtime/prime/bin/startup-check.sh +1 -1
- package/runtime/prime/bin/telemetry.sh +11 -4
- package/runtime/prime/bin/test-baseline.sh +33 -5
- package/runtime/prime/bin/todo.sh +1 -1
- package/runtime/prime/bin/worktree-create.sh +15 -45
- package/runtime/prime/bin/worktree.sh +6 -0
- package/runtime/prime/docs/spirit-discovery.md +96 -0
- package/runtime/prime/egregore_runtime/__init__.py +18 -12
- package/runtime/prime/egregore_runtime/access.py +608 -0
- package/runtime/prime/egregore_runtime/access_cli.py +190 -0
- package/runtime/prime/egregore_runtime/adapters/invitation.py +5 -3
- package/runtime/prime/egregore_runtime/adapters/qmd.py +521 -47
- package/runtime/prime/egregore_runtime/admin_entry.py +56 -0
- package/runtime/prime/egregore_runtime/artifacts.py +222 -16
- package/runtime/prime/egregore_runtime/contracts.py +27 -0
- package/runtime/prime/egregore_runtime/eligibility.py +5 -12
- package/runtime/prime/egregore_runtime/episode_binding.py +16 -5
- package/runtime/prime/egregore_runtime/github_issues.py +2 -2
- package/runtime/prime/egregore_runtime/handoff_lookup.py +9 -20
- package/runtime/prime/egregore_runtime/harness_cli.py +674 -149
- package/runtime/prime/egregore_runtime/held.py +2436 -0
- package/runtime/prime/egregore_runtime/hosted_api.py +137 -0
- package/runtime/prime/egregore_runtime/identity.py +196 -65
- package/runtime/prime/egregore_runtime/identity_transfer.py +6 -2
- package/runtime/prime/egregore_runtime/ingest_cli.py +2 -2
- package/runtime/prime/egregore_runtime/ingest_index_cli.py +4 -16
- package/runtime/prime/egregore_runtime/installation.py +119 -0
- package/runtime/prime/egregore_runtime/investigation.py +64 -41
- package/runtime/prime/egregore_runtime/investigation_guidance.py +16 -11
- package/runtime/prime/egregore_runtime/issue_cli.py +51 -32
- package/runtime/prime/egregore_runtime/issue_reporting.py +4 -3
- package/runtime/prime/egregore_runtime/issue_triage_service.py +2 -2
- package/runtime/prime/egregore_runtime/issues.py +12 -6
- package/runtime/prime/egregore_runtime/knowledge.py +331 -5
- package/runtime/prime/egregore_runtime/knowledge_cli.py +57 -17
- package/runtime/prime/egregore_runtime/lifecycle.py +88 -22
- package/runtime/prime/egregore_runtime/lifecycle_cli.py +21 -13
- package/runtime/prime/egregore_runtime/line_ids.py +10 -0
- package/runtime/prime/egregore_runtime/memory_push.py +349 -0
- package/runtime/prime/egregore_runtime/observe.py +64 -25
- package/runtime/prime/egregore_runtime/observers.py +19 -0
- package/runtime/prime/egregore_runtime/optional_modules.py +13 -0
- package/runtime/prime/egregore_runtime/outputs_cli.py +321 -0
- package/runtime/prime/egregore_runtime/policy.py +651 -24
- package/runtime/prime/egregore_runtime/private_result.py +2 -1
- package/runtime/prime/egregore_runtime/projection.py +946 -0
- package/runtime/prime/egregore_runtime/quest_cli.py +20 -23
- package/runtime/prime/egregore_runtime/question_cli.py +15 -11
- package/runtime/prime/egregore_runtime/questions.py +10 -10
- package/runtime/prime/egregore_runtime/quests.py +10 -10
- package/runtime/prime/egregore_runtime/record_publish.py +548 -0
- package/runtime/prime/egregore_runtime/record_subject.py +564 -0
- package/runtime/prime/egregore_runtime/research_ingest.py +2 -4
- package/runtime/prime/egregore_runtime/runtime.py +169 -60
- package/runtime/prime/egregore_runtime/sandbox.py +194 -0
- package/runtime/prime/egregore_runtime/scroll_cli.py +22 -32
- package/runtime/prime/egregore_runtime/scrolls.py +23 -15
- package/runtime/prime/egregore_runtime/shadow.py +262 -0
- package/runtime/prime/egregore_runtime/spirit_discovery.py +487 -0
- package/runtime/prime/egregore_runtime/spirit_discovery_cli.py +25 -0
- package/runtime/prime/egregore_runtime/spirit_discovery_transport.py +18 -0
- package/runtime/prime/egregore_runtime/status.py +101 -44
- package/runtime/prime/egregore_runtime/status_cli.py +13 -5
- package/runtime/prime/egregore_runtime/sync.py +39 -21
- package/runtime/prime/egregore_runtime/taste.py +934 -0
- package/runtime/prime/egregore_runtime/taste_cli.py +242 -0
- package/runtime/prime/egregore_runtime/taste_rules.py +91 -0
- package/runtime/prime/egregore_runtime/telemetry.py +26 -3
- package/runtime/prime/egregore_runtime/todo_cli.py +15 -12
- package/runtime/prime/egregore_runtime/todos.py +3 -2
- package/runtime/prime/egregore_runtime/upgrade.py +85 -58
- package/runtime/prime/egregore_runtime/writeback.py +330 -24
- package/runtime/prime/manifest.json +66 -5
- package/runtime/claude/bin/thread.sh +0 -9
- package/runtime/claude/egregore_runtime/admin_gate.py +0 -102
- package/runtime/claude/egregore_runtime/thread_cli.py +0 -173
- package/runtime/claude/egregore_runtime/threads.py +0 -521
- package/runtime/codex/bin/thread.sh +0 -9
- package/runtime/codex/egregore_runtime/admin_gate.py +0 -102
- package/runtime/codex/egregore_runtime/thread_cli.py +0 -173
- package/runtime/codex/egregore_runtime/threads.py +0 -521
- package/runtime/pi/bin/thread.sh +0 -9
- package/runtime/pi/egregore_runtime/admin_gate.py +0 -102
- package/runtime/pi/egregore_runtime/thread_cli.py +0 -173
- package/runtime/pi/egregore_runtime/threads.py +0 -521
- package/runtime/prime/bin/thread.sh +0 -9
- package/runtime/prime/egregore_runtime/admin_gate.py +0 -102
- package/runtime/prime/egregore_runtime/thread_cli.py +0 -173
- package/runtime/prime/egregore_runtime/threads.py +0 -521
|
@@ -0,0 +1,2436 @@
|
|
|
1
|
+
"""Held material: the classes a non-admin is never handed, decided in one place.
|
|
2
|
+
|
|
3
|
+
``admin: true`` frontmatter gives a canonical document its documented
|
|
4
|
+
meaning: hidden on Egregore surfaces from non-admin members, and refused for
|
|
5
|
+
external publication without an authorized admin's separate confirmation.
|
|
6
|
+
|
|
7
|
+
This is soft product-layer protection. Every organization member holds full
|
|
8
|
+
repository access and can read raw Git files; nothing here claims per-document
|
|
9
|
+
filesystem confidentiality, and no surface may describe it as an ACL. What it
|
|
10
|
+
guarantees is that Egregore itself — retrieval evidence, Observe context,
|
|
11
|
+
source opening, listings, rendering, publication — never hands admin-marked
|
|
12
|
+
content to a non-admin member, exposing at most an opaque withheld count.
|
|
13
|
+
|
|
14
|
+
Admin capability is decided by ``policy.is_admin`` from the roster that
|
|
15
|
+
``access.authorization_inputs`` reads; ``policy.check`` applies every class.
|
|
16
|
+
Every hold rule and the admin derivatives moved here from the Mind exporter,
|
|
17
|
+
which asks for them; its curation stays there.
|
|
18
|
+
|
|
19
|
+
Redaction lives here too. ``package`` is how memory text leaves the
|
|
20
|
+
Runtime: ``allows`` on every path an item names, then ``redact`` strips the
|
|
21
|
+
names of the documents held from a non-admin reader in a scope that applies
|
|
22
|
+
every class (a hand-over; a read under roster-v1). ``Redactor`` finds those
|
|
23
|
+
names for the projection and for reads alike; the exporter strips its own
|
|
24
|
+
withheld set with ``strip_names``.
|
|
25
|
+
|
|
26
|
+
``verdicts`` classifies a memory tree. Its cache,
|
|
27
|
+
``cache_path``, in this user's home, can be written by any process of
|
|
28
|
+
this user, so it is a hint for the operator's own reads and slot only: every
|
|
29
|
+
other principal recomputes with ``trust_cache=False``.
|
|
30
|
+
|
|
31
|
+
``tree`` with that trust also keeps a verdict index beside the hint
|
|
32
|
+
(``index_path``): the verdicts of one walk, by hashed path and content
|
|
33
|
+
digest, under the digest of every document's path, size, times and inode.
|
|
34
|
+
A later process whose walk matches reads it back instead of reading and
|
|
35
|
+
parsing memory again; a file changed shortly before that walk is hashed
|
|
36
|
+
again first (``RACY_NS``). Builds of one memory take turns, so each memory
|
|
37
|
+
state is built once per machine.
|
|
38
|
+
"""
|
|
39
|
+
|
|
40
|
+
from __future__ import annotations
|
|
41
|
+
|
|
42
|
+
import atexit
|
|
43
|
+
from contextlib import contextmanager
|
|
44
|
+
from contextvars import ContextVar
|
|
45
|
+
from dataclasses import dataclass, fields, is_dataclass, replace
|
|
46
|
+
import fcntl
|
|
47
|
+
from functools import lru_cache
|
|
48
|
+
import hashlib
|
|
49
|
+
import html
|
|
50
|
+
import json
|
|
51
|
+
import math
|
|
52
|
+
import os
|
|
53
|
+
from pathlib import Path
|
|
54
|
+
import re
|
|
55
|
+
import stat
|
|
56
|
+
import subprocess
|
|
57
|
+
import time
|
|
58
|
+
from collections import Counter
|
|
59
|
+
from typing import Any, Callable, Iterable, Mapping
|
|
60
|
+
import unicodedata
|
|
61
|
+
from urllib.parse import unquote
|
|
62
|
+
|
|
63
|
+
from .artifacts import ArtifactSchemaError, artifact_id_from_path, canonical_artifact_id, split_frontmatter
|
|
64
|
+
from .errors import EgregoreRuntimeError
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
class BoundedReadUnavailable(ValueError):
|
|
68
|
+
"""No partial hold projection is an authority result."""
|
|
69
|
+
|
|
70
|
+
|
|
71
|
+
@dataclass(frozen=True, slots=True)
|
|
72
|
+
class HoldReadLimits:
|
|
73
|
+
entries: int = 16384
|
|
74
|
+
documents: int = 4096
|
|
75
|
+
dependency_bytes: int = 256 * 1024
|
|
76
|
+
total_bytes: int = 32 * 1024 * 1024
|
|
77
|
+
inventory_bytes: int = 4 * 1024 * 1024
|
|
78
|
+
path_bytes: int = 4096
|
|
79
|
+
link_hops: int = 40
|
|
80
|
+
passes: int = 64
|
|
81
|
+
work: int = 256 * 1024 * 1024
|
|
82
|
+
settings_bytes: int = 64 * 1024
|
|
83
|
+
total_settings_bytes: int = 256 * 1024
|
|
84
|
+
selected_bytes: int = 8 * 1024 * 1024
|
|
85
|
+
seconds: float = 8.0
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
DEFAULT_HOLD_READ_LIMITS = HoldReadLimits()
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
class _HoldReadBudget:
|
|
92
|
+
__slots__ = ("limits", "deadline", "counts", "rechecking", "invalid", "authority")
|
|
93
|
+
|
|
94
|
+
def __init__(self, limits, deadline):
|
|
95
|
+
self.limits, self.deadline = limits, deadline
|
|
96
|
+
self.counts, self.rechecking, self.invalid, self.authority = {}, False, False, {}
|
|
97
|
+
|
|
98
|
+
def fail(self):
|
|
99
|
+
self.invalid = True
|
|
100
|
+
raise BoundedReadUnavailable("bounded read unavailable")
|
|
101
|
+
|
|
102
|
+
def remaining(self):
|
|
103
|
+
if self.invalid or (remaining := self.deadline - time.monotonic()) <= 0:
|
|
104
|
+
self.fail()
|
|
105
|
+
return remaining
|
|
106
|
+
|
|
107
|
+
def charge(self, name, amount=1):
|
|
108
|
+
self.remaining()
|
|
109
|
+
if type(amount) is not int or amount < 0:
|
|
110
|
+
self.fail()
|
|
111
|
+
ceiling = getattr(self.limits, name)
|
|
112
|
+
if name in {"total_bytes", "selected_bytes"} and not self.rechecking:
|
|
113
|
+
ceiling //= 2
|
|
114
|
+
self.counts[name] = self.counts.get(name, 0) + amount
|
|
115
|
+
if self.counts[name] > ceiling:
|
|
116
|
+
self.fail()
|
|
117
|
+
|
|
118
|
+
def recheck(self):
|
|
119
|
+
self.remaining()
|
|
120
|
+
if self.rechecking:
|
|
121
|
+
self.fail()
|
|
122
|
+
self.rechecking = True
|
|
123
|
+
|
|
124
|
+
def bind_authority(self, key, witness):
|
|
125
|
+
self.remaining()
|
|
126
|
+
if key in self.authority and self.authority[key] != witness:
|
|
127
|
+
self.fail()
|
|
128
|
+
self.authority[key] = witness
|
|
129
|
+
|
|
130
|
+
|
|
131
|
+
def new_read_budget(*, limits=DEFAULT_HOLD_READ_LIMITS, deadline_monotonic=None):
|
|
132
|
+
if type(limits) is not HoldReadLimits:
|
|
133
|
+
raise BoundedReadUnavailable("bounded read unavailable")
|
|
134
|
+
for item in fields(limits):
|
|
135
|
+
value = getattr(limits, item.name)
|
|
136
|
+
valid = (type(value) in (int, float) and math.isfinite(value) if item.name == "seconds"
|
|
137
|
+
else type(value) is int)
|
|
138
|
+
if not valid or not 0 < value <= getattr(DEFAULT_HOLD_READ_LIMITS, item.name):
|
|
139
|
+
raise BoundedReadUnavailable("bounded read unavailable")
|
|
140
|
+
now = time.monotonic()
|
|
141
|
+
if deadline_monotonic is not None and (type(deadline_monotonic) not in (int, float)
|
|
142
|
+
or not math.isfinite(deadline_monotonic)):
|
|
143
|
+
raise BoundedReadUnavailable("bounded read unavailable")
|
|
144
|
+
budget = _HoldReadBudget(limits, min(now + limits.seconds, deadline_monotonic) if deadline_monotonic is not None else now + limits.seconds)
|
|
145
|
+
budget.remaining()
|
|
146
|
+
return budget
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def _read_budget(budget):
|
|
150
|
+
if type(budget) is not _HoldReadBudget:
|
|
151
|
+
raise BoundedReadUnavailable("bounded read unavailable")
|
|
152
|
+
budget.remaining()
|
|
153
|
+
return budget
|
|
154
|
+
|
|
155
|
+
|
|
156
|
+
_WORK_BUDGET = ContextVar("egregore_bounded_hold_work", default=None)
|
|
157
|
+
|
|
158
|
+
|
|
159
|
+
def _work(value=0):
|
|
160
|
+
if (budget := _WORK_BUDGET.get()) is not None:
|
|
161
|
+
budget.charge("work", 1 + (len(value) if isinstance(value, (str, bytes)) else value))
|
|
162
|
+
|
|
163
|
+
_FRONTMATTER_ADMIN = re.compile(r"^\s*admin:\s*[\"']?true[\"']?\s*$", re.MULTILINE)
|
|
164
|
+
_HEAD_BYTES = 4096
|
|
165
|
+
try: # this module and the two a verdict reads documents through: frontmatter, and the map's internal events
|
|
166
|
+
_SOURCE = b"".join((Path(__file__).parent / name).read_bytes() for name in ("held.py", "artifacts.py", "eligibility.py"))
|
|
167
|
+
except OSError:
|
|
168
|
+
_SOURCE = b""
|
|
169
|
+
# Any edit to these modules re-checks every cached verdict, here, in the verdict index and in the exporter's
|
|
170
|
+
# parse cache.
|
|
171
|
+
HELD_RULES_VERSION = "held-rules/2:" + hashlib.sha256(_SOURCE).hexdigest()[:16]
|
|
172
|
+
|
|
173
|
+
|
|
174
|
+
def document_is_admin(text: str) -> bool:
|
|
175
|
+
"""True when the document's YAML frontmatter declares ``admin: true``."""
|
|
176
|
+
|
|
177
|
+
if "\r" in text: # CRLF or bare CR line ends read as on disk
|
|
178
|
+
text = text.replace("\r\n", "\n").replace("\r", "\n")
|
|
179
|
+
if not text.startswith("---"):
|
|
180
|
+
return False
|
|
181
|
+
end = text.find("\n---", 3)
|
|
182
|
+
head = text[3:end] if end != -1 else text[3:_HEAD_BYTES]
|
|
183
|
+
return bool(_FRONTMATTER_ADMIN.search(head))
|
|
184
|
+
|
|
185
|
+
|
|
186
|
+
# Class 1, then the rule classes in the exporter's order (noise, curation, sits between 2 and 3 there).
|
|
187
|
+
CLASS_NAMES = {1: "admin: true", 2: "paperwork", 3: "personnel or equity", 4: "raw transcript", 5: "fundraising",
|
|
188
|
+
6: "restriction", 7: "private prose", 8: "personnel change", 9: "derived from an admin document"}
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def text_classes(text: str, rel: str | None = None) -> tuple[int, ...]:
|
|
192
|
+
"""Class 1, or the first rule's class at memory path ``rel`` (no path: class 1 only; derivatives:
|
|
193
|
+
``Tree.classes``)."""
|
|
194
|
+
if document_is_admin(text):
|
|
195
|
+
return (1,)
|
|
196
|
+
name = _rule(rel, text) if rel is not None else None
|
|
197
|
+
return (RULE_CLASS[name],) if name else ()
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def blob_classes(data: bytes, rel: str | None = None) -> tuple[int, ...]:
|
|
201
|
+
"""Classify committed bytes exactly as the same file on disk, and the rules as the exporter reads it."""
|
|
202
|
+
text = data.decode("utf-8", "replace")
|
|
203
|
+
if rel is None:
|
|
204
|
+
return text_classes(text.replace("\r\n", "\n").replace("\r", "\n")[:_HEAD_BYTES])
|
|
205
|
+
return text_classes(text, rel)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
def admin_class(data: bytes) -> tuple[int, ...]:
|
|
209
|
+
"""Class 1 of a document's whole bytes, read as ``Tree.classes`` reads it: (1,) or ()."""
|
|
210
|
+
return (1,) if document_is_admin(data.decode("utf-8", errors="replace")) else ()
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def classes(path: Path) -> tuple[int, ...]:
|
|
214
|
+
"""Classify a regular file; missing or unreadable files have no known hold."""
|
|
215
|
+
try:
|
|
216
|
+
if path.is_file():
|
|
217
|
+
with path.open(encoding="utf-8", errors="replace") as handle:
|
|
218
|
+
return text_classes(handle.read(_HEAD_BYTES))
|
|
219
|
+
except (OSError, RuntimeError, ValueError):
|
|
220
|
+
pass
|
|
221
|
+
return ()
|
|
222
|
+
|
|
223
|
+
|
|
224
|
+
def held_class(path: Path) -> int | None:
|
|
225
|
+
return next(iter(classes(path)), None)
|
|
226
|
+
|
|
227
|
+
|
|
228
|
+
LENIENT = frozenset({"all-access-v1", "roster-v1-shadow"}) # the read policies that hold class 1 alone, unredacted
|
|
229
|
+
|
|
230
|
+
|
|
231
|
+
def judge(scope: str, policy: str, found: tuple[int, ...], released: bool) -> tuple[int | None, int | None]:
|
|
232
|
+
"""(the class that holds, the class roster-v1 alone would hold) for a non-admin.
|
|
233
|
+
|
|
234
|
+
A hand-over applies every class under every policy; reads apply class 1
|
|
235
|
+
under all-access-v1 and roster-v1-shadow, and every class otherwise.
|
|
236
|
+
``access.release`` lifts rule classes, never class 1.
|
|
237
|
+
"""
|
|
238
|
+
live = [number for number in found if number == 1 or not released]
|
|
239
|
+
lenient = scope == "read" and policy in LENIENT
|
|
240
|
+
hold = next((number for number in live if number == 1 or not lenient), None)
|
|
241
|
+
shadow = next((number for number in live if number != 1), None) if policy == "roster-v1-shadow" and scope == "read" else None
|
|
242
|
+
return hold, None if hold else shadow
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
class _lazy:
|
|
246
|
+
"""A pattern compiled on first use, so importing the Runtime does not compile every rule."""
|
|
247
|
+
|
|
248
|
+
__slots__ = ("_args", "_pattern")
|
|
249
|
+
|
|
250
|
+
def __init__(self, *args):
|
|
251
|
+
self._args, self._pattern = args, None
|
|
252
|
+
|
|
253
|
+
def __getattr__(self, name):
|
|
254
|
+
if self._pattern is None:
|
|
255
|
+
self._pattern = re.compile(*self._args)
|
|
256
|
+
return getattr(self._pattern, name)
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
@lru_cache(maxsize=65536)
|
|
260
|
+
def fold(text: str) -> str:
|
|
261
|
+
"""Casefold and strip accents (Çağrı -> cagri, Öztürk -> ozturk)."""
|
|
262
|
+
text = str(text)
|
|
263
|
+
if not text.isascii():
|
|
264
|
+
text = unicodedata.normalize("NFKD", text)
|
|
265
|
+
text = text.translate({ord(ch):None for ch in set(text) if unicodedata.combining(ch)})
|
|
266
|
+
return " ".join(text.casefold().replace("ı", "i").split())
|
|
267
|
+
|
|
268
|
+
|
|
269
|
+
# -- The hold rules, moved from experiments/egregore-mind/export.py at 45257e4b ----------------
|
|
270
|
+
|
|
271
|
+
# Each rule describes a record, not an incidental word in a discussion's body.
|
|
272
|
+
PAPERWORK_RULES = [(name, _lazy(pattern, re.I)) for name, pattern in (
|
|
273
|
+
("tax-record", r"\b(?:tax (?:returns?|assessments?|filings?|paperwork|records?|forms?)|steuer(?:bescheid|erklarung|erklärung))\b"),
|
|
274
|
+
("tax-claim", r"\b(?:tax|tax credit|research credit|fzul|forschungszulage)\b.*\b(?:claims?|follow up|correspondence|filing|application|evidence|nachforderung)\b|\b(?:claims?|follow up|correspondence|nachforderung)\b.*\b(?:tax|fzul|forschungszulage)\b"),
|
|
275
|
+
("tax-credit-submission", r"\b(?:fzul|forschungszulage|research tax credit)\b.*\b(?:20\d\d|workbook|answers?|submission|einreichfassung|fassung|review)\b"),
|
|
276
|
+
("insurance-record", r"\b(?:insurance|krankenkasse)\b.*\b(?:policy|policies|claim|premium|reimbursement|correspondence|correction|contacts?|details|paperwork)\b|\b(?:policy|claim|contacts?|correspondence|paperwork)\b.*\b(?:insurance|krankenkasse)\b"),
|
|
277
|
+
("invoice-record", r"\b(?:invoice (?:\d+|number|amount|payment|record|paperwork)|(?:vendor|supplier|customer|paid|unpaid) invoice)\b"),
|
|
278
|
+
("bank-record", r"\bbank (?:statement|details|transfer receipt|paperwork)\b|\b(?:company|personal|corporate) bank account\b"),
|
|
279
|
+
("registration-record", r"\b(?:(?:company|corporate|business) registration|registration (?:certificate|details|number|paperwork)|commercial register)\b"),
|
|
280
|
+
("payroll-record", r"\b(?:payroll|salary|compensation)\b.*\b(?:statement|record|slip|payment|paperwork|ledger)\b|\b(?:payslips?|offer letters?|cap tables?|vesting schedule)\b"),
|
|
281
|
+
("signed-contract", r"\b(?:signed|executed|countersigned)\b.*\b(?:contracts?|agreements?|nda)\b|\b(?:contracts?|agreements?|nda)\b.*\b(?:signed|executed|counterparties|paperwork)\b|\b(?:employment contract|non disclosure agreement)\b"),
|
|
282
|
+
("private-contact-list", r"\b(?:vc|investor) (?:contact|intro|introduction)s?\b.*\b(?:list|shortlist|questions)\b|\b(?:list|shortlist)\b.*\b(?:vc|investor) contacts?\b"),
|
|
283
|
+
("personal-administration", r"\bops todos\b|\b(?:private|personal)\b.*\bletter parser\b|\bletter parser\b.*\b(?:private|personal|insurance|krankenkasse)\b"),
|
|
284
|
+
("correspondence-processing", r"\bletter (?:parser|processing) skill\b"),
|
|
285
|
+
)]
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def paperwork_rule(fm, rel):
|
|
289
|
+
subject = " ".join(str(fm.get(key) or "") for key in ("title", "topics")) + " " + rel
|
|
290
|
+
subject = " ".join(re.sub(r"[^\w]+", " ", fold(subject)).split())
|
|
291
|
+
return next((name for name, pattern in PAPERWORK_RULES if pattern.search(subject)), None)
|
|
292
|
+
|
|
293
|
+
|
|
294
|
+
# The members hold approved on Oct 9 (the SENSITIVE rule of that day, word for word): personnel and
|
|
295
|
+
# equity material is held wherever a document mentions it (frontmatter, body, title, summary or
|
|
296
|
+
# path), not only when its subject names a record. The paperwork rules above still apply as well.
|
|
297
|
+
PERSONNEL = _lazy(
|
|
298
|
+
r"\b(?:equity|valuations?|compensation|vesting|vested|payroll|salary|salaries|offer letters?|"
|
|
299
|
+
r"cap tables?|stock options?|share options?|esop|severance|"
|
|
300
|
+
r"comp (?:mechanics?|packages?|negotiations?|terms|structure))\b", re.IGNORECASE)
|
|
301
|
+
|
|
302
|
+
|
|
303
|
+
def personnel_rule(text, title="", summary="", rel=""):
|
|
304
|
+
subject = " ".join((text, title, summary, re.sub(r"[-_/.]+", " ", rel)))
|
|
305
|
+
return "personnel-or-equity" if PERSONNEL.search(subject) else None
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
# Personnel-adjacent material, held by meaning since 2026-10-12 (third audit): who joins, who leaves,
|
|
309
|
+
# what investors decided and conflicts between people. The vocabulary is deliberately narrow and was
|
|
310
|
+
# reviewed against the corpus: each phrase names a person's move or an investor's outcome, not the
|
|
311
|
+
# words in ordinary product prose ("user departure" settings, "leave Egregore and take your files",
|
|
312
|
+
# "results fall out of the analysis", "a recruiting note" template).
|
|
313
|
+
_NAME = r"[A-Z][A-Za-z’'-]+"
|
|
314
|
+
_SUBJECT = r"(?:" + _NAME + r"|[Hh]e|[Ss]he|I|[Tt]hey|[Ww]e)"
|
|
315
|
+
PERSONNEL_CHANGE = _lazy("|".join((
|
|
316
|
+
r"\b(?:he|she|they|we|I|[A-Z][A-Za-z’'-]+)\s+(?:(?:is|was|are|were|has|had|have|will|be)\s+){0,3}"
|
|
317
|
+
r"(?:leave|leaving|left|leaves|quit|quits|resign(?:ed|s|ing)?\s+from)\s+(?:the|our)\s+(?:company|team)\b",
|
|
318
|
+
r"\bfounding[- ](?:team |role |equity )?offers?\b",
|
|
319
|
+
r"\b(?:job|employment|hiring) offers?\b",
|
|
320
|
+
r"(?-i:\boffer(?:ed|ing|s)?[ \t]+(?:him|her|them|[A-Z][a-z]+)[ \t]+(?:a[ \t]+|the[ \t]+)?"
|
|
321
|
+
r"(?:role|position|seat|job|co-?founder\w*|founding[ \t]+\w+|partnership)\b)",
|
|
322
|
+
r"\bjoin(?:s|ed|ing)?\s+(?:us\s+|the team\s+|the company\s+)?as\s+(?:a\s+|an\s+|our\s+|the\s+)?"
|
|
323
|
+
r"(?:co-?founder|founding \w+|cto|ceo|coo|cpo|cfo|chief \w+|head of \w+|vp|full[- ]time \w+|"
|
|
324
|
+
r"part[- ]time \w+|employee|first hire)\b",
|
|
325
|
+
r"\brecruit(?:ment|ing) (?:conversation|call|pitch|attempt|offer)s?\b",
|
|
326
|
+
r"\b(?:hir(?:e|ed|es|ing)|recruit(?:ed|s|ing)?)[ \t]+(?-i:" + _NAME + r")\b",
|
|
327
|
+
r"\b(?-i:" + _SUBJECT + r")\s+(?:(?:is|was|are|were|has|had|have|will|be)\s+){0,3}(?:leave|leaving|left|leaves|quit|quits|resign(?:ed|s|ing)?\s+from)\s+"
|
|
328
|
+
r"(?:the company|the team|our company|our team|(?-i:" + _NAME + r"(?:[ \t]+" + _NAME + r")*))\b",
|
|
329
|
+
r"\b(?-i:" + _SUBJECT + r")\s+(?:is\s+|was\s+|will\s+|has\s+)?step(?:s|ped|ping)?\s+(?:down|back)\s+from\b",
|
|
330
|
+
r"\b(?-i:" + _NAME + r")['’]s\s+(?:departure|resignation)\b",
|
|
331
|
+
r"\b(?:departure|resignation) of (?:a |the |our )?(?:co-?)?founder\b",
|
|
332
|
+
r"\b(?:is|was|are|were|currently|temporarily)\s+(?:currently\s+|temporarily\s+)?between jobs\b",
|
|
333
|
+
)), re.I)
|
|
334
|
+
# A founder handover is held by its title, summary or kind, not by a closing note that mentions one.
|
|
335
|
+
FOUNDER_HANDOVER = _lazy(r"\b(?:co-?)?founder[- ]hand(?:over|off)\b", re.I)
|
|
336
|
+
INVESTOR_OUTCOME = _lazy("|".join((
|
|
337
|
+
r"\b(?:passed|declined|rejected|pass(?:es)?|declines?) on (?:the |our |this |us )?(?:current |seed |pre-seed )?"
|
|
338
|
+
r"(?:round|raise|deal|investment)\b",
|
|
339
|
+
r"\b(?:decided|chose|decides|choosing) (?:not to|against) invest\w*\b",
|
|
340
|
+
r"\b(?:investor|vc|fund|angel)s?['’]s?\s+(?:rejection|pass|decline|commitment)\b",
|
|
341
|
+
r"\b(?:investors?|vcs?|fund|angels?|lead)\s+(?:has\s+|have\s+|had\s+)?(?:passed|declined|turned (?:us )?down|committed)\b",
|
|
342
|
+
r"\bcommitted (?:to|in) (?:the|our|this) (?:round|raise)\b",
|
|
343
|
+
)), re.I)
|
|
344
|
+
# "<Name>'s rejection" counts in a document that speaks of investing at all.
|
|
345
|
+
NAMED_REJECTION = _lazy(r"\b[A-Z][A-Za-z0-9&]+['’]s\s+(?:rejection|pass)\b")
|
|
346
|
+
INVESTING = _lazy(r"\binvest\w*|\bVCs?\b|\bfundrais\w*|\b(?:seed|pre-seed) round\b", re.I)
|
|
347
|
+
INVESTOR_KINDS = {"investor-decision", "investor-outcome", "investor_decision", "investor_outcome"}
|
|
348
|
+
INTERPERSONAL = _lazy("|".join((
|
|
349
|
+
r"\b(?:co-?founder|business partner|ex-?partner)s?\s+confront(?:ed|s|ing)?\s+\w+\b",
|
|
350
|
+
r"\bconfront(?:ed|s|ing)?\s+(?:him|her|them|me|us)\b",
|
|
351
|
+
r"\bconfront(?:ed|s|ing)?\s+(?-i:" + _NAME + r")\b",
|
|
352
|
+
r"\bfalling[- ]out (?:with|between)\b|\bfell out with\b|\bhad a falling[- ]out\b",
|
|
353
|
+
r"\b(?:conflict|dispute|fight|tension)s? (?:with|between) (?:his|her|my|our|their|the) "
|
|
354
|
+
r"(?:co-?founders?|business partners?|ex-?partners?)\b",
|
|
355
|
+
)), re.I)
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def personnel_change_rule(text, title="", summary="", fm=None):
|
|
359
|
+
"""personnel-change, investor-outcome, interpersonal-conflict, or None."""
|
|
360
|
+
fm = fm or {}
|
|
361
|
+
kinds = {str(fm.get(key) or "").strip().lower() for key in ("kind", "type", "record_type")}
|
|
362
|
+
subject = "\n".join((title or "", summary or "", text or ""))
|
|
363
|
+
if (PERSONNEL_CHANGE.search(subject) or kinds & {"founder-handover", "founder_handover"}
|
|
364
|
+
or FOUNDER_HANDOVER.search("\n".join((title or "", summary or "")))):
|
|
365
|
+
return "personnel-change"
|
|
366
|
+
if (kinds & INVESTOR_KINDS or INVESTOR_OUTCOME.search(subject)
|
|
367
|
+
or (NAMED_REJECTION.search(subject) and INVESTING.search(subject))):
|
|
368
|
+
return "investor-outcome"
|
|
369
|
+
if INTERPERSONAL.search(subject):
|
|
370
|
+
return "interpersonal-conflict"
|
|
371
|
+
return None
|
|
372
|
+
|
|
373
|
+
|
|
374
|
+
# The members hold extended on 2026-10-12 (owner decision after the second privacy audit). Like the
|
|
375
|
+
# personnel rule it reads the whole document, but it holds by meaning, not by a single word:
|
|
376
|
+
# valuation-or-dilution: the instruments and terms of a raise (pre-/post-money, valuation cap,
|
|
377
|
+
# convertible note, a SAFE as an uppercase word in a funding phrase, term sheet, cap table,
|
|
378
|
+
# dilution of ownership), an equity percentage ("10% of the company", "~10% dilution"), or a
|
|
379
|
+
# raise-sized amount within a few words of a percentage on a line about the raise;
|
|
380
|
+
# fundraising-figures: a raise-sized amount within a few words of the raise, a soft commitment,
|
|
381
|
+
# an investor's fund, ticket or check size.
|
|
382
|
+
# A raise-sized amount carries a currency and is at least 100,000 ("$4M", "€500k–3M", "$4,000,000");
|
|
383
|
+
# prices, per-user costs and token counts are not.
|
|
384
|
+
MONEY = _lazy(
|
|
385
|
+
r"(?:[$€£]\s?~?(\d[\d,.]*)(?:\s?([kKmMbB]n?|million|billion)\b)?"
|
|
386
|
+
r"(?:\s?(?:[–-]|to)\s?[$€£]?(\d[\d,.]*)(?:\s?([kKmMbB]n?|million|billion)\b)?)?"
|
|
387
|
+
r"|\b(\d[\d,.]*)(?:\s?([kKmMbB]n?|million|billion))?\s?(?:USD|EUR|GBP|dollars|euros)\b)")
|
|
388
|
+
_SCALE = {"k": 1e3, "m": 1e6, "mn": 1e6, "million": 1e6, "b": 1e9, "bn": 1e9, "billion": 1e9}
|
|
389
|
+
RAISE_SIZED = 100_000
|
|
390
|
+
PERCENT = _lazy(r"[<>~≈]?\s?\d+(?:[.,]\d+)?\s?%")
|
|
391
|
+
EQUITY_TERMS = _lazy(
|
|
392
|
+
r"\b(?:pre|post)[- ]?money\b|\bvaluation caps?\b|\bconvertible (?:notes?|loans?)\b|\bterm[- ]?sheets?\b|"
|
|
393
|
+
r"\bcap[- ]?tables?\b|\banti[- ]?dilution\b|"
|
|
394
|
+
r"\bdilut\w*\s+(?:(?:our|the|their|founders?['’]?|existing)\s+)?(?:ownership|equity|stakes?|shares?|shareholders?|founders?)\b|"
|
|
395
|
+
r"\d(?:[.,]\d+)?\s?%\s+(?:dilution|of (?:the|our) (?:company|business)|(?:equity )?stake|equity)\b|"
|
|
396
|
+
r"\bdilution\s+(?:of\s+)?(?:about\s+|around\s+|roughly\s+|under\s+|below\s+|~|<)?\s*\d+(?:[.,]\d+)?\s?%|"
|
|
397
|
+
r"\b(?:sell|selling|sold|give up|giving up)\s+(?:about\s+|around\s+|roughly\s+|~)?\d+(?:[.,]\d+)?\s?%", re.I)
|
|
398
|
+
# "SAFE" as the instrument: uppercase, in a funding phrase ("on a SAFE", "SAFE note"), or next to an amount.
|
|
399
|
+
SAFE_TERM = _lazy(r"\b(?:[Aa]n?|[Tt]he|on|via|uncapped|capped|check|angel)\s+SAFEs?\b|"
|
|
400
|
+
r"\bSAFEs?\s+(?:notes?|rounds?|raise|agreements?|investments?|terms|caps?|financing|instruments?)\b")
|
|
401
|
+
SAFE_WORD = _lazy(r"\bSAFEs?\b")
|
|
402
|
+
RAISE_CONTEXT = _lazy(r"\bSAFEs?\b|(?i:(?<![\w-])(?:dilut\w*|equity|stakes?|valuation|valued|raise|raising|rounds?|"
|
|
403
|
+
r"ask|sold|selling|pre-?seed|(?:pre|post)[- ]?money)(?![\w-])|\bof (?:the|our) company\b)")
|
|
404
|
+
# "raised" is left out: it mostly reports another company's past round ("X raised $8M"). "seed" counts
|
|
405
|
+
# only as our round ("$4M seed →", "$240k seed for…", "seed capital"), not "$2.6M seed (their investors)".
|
|
406
|
+
# "fundraising" itself and investor or VC wording count as well (2026-10-12, third audit); "funds"
|
|
407
|
+
# does not, because "a $2.6M seed (two funds)" reports another company's round.
|
|
408
|
+
FUNDRAISING_TERM = _lazy(
|
|
409
|
+
r"(?<![\w-])(?:soft[- ]?commit\w*|tickets?|checks?|check size|sweet spot|rounds?|raise|raising|pre-?seed|"
|
|
410
|
+
r"seed(?=[\s*_]*(?:→|->|(?:round|raise|capital|funding|for)\b))|fund|ask|lead investor|fundrais\w*|"
|
|
411
|
+
r"investors?|VCs?|angels?)(?![\w-])", re.I)
|
|
412
|
+
FEW_WORDS = _lazy(r"[^\W_]+")
|
|
413
|
+
CONTEXT_WORDS = 6
|
|
414
|
+
# An investor's fund size within one clause of the amount ("eight VC inbounds: pre-seed funds plus
|
|
415
|
+
# Alpha ($1B, Berlin)"): a raise-sized amount and investor wording in the same clause. Clauses end
|
|
416
|
+
# at sentence or clause punctuation, a spaced dash, a line or a table cell. Bare "fund", "funds" and
|
|
417
|
+
# "seed" are not investor wording here: "willingness to fund this work", "a $2.6M seed (two funds)".
|
|
418
|
+
INVESTOR_WORDING = _lazy(
|
|
419
|
+
r"(?<![\w.-])(?:VCs?|investors?|angels?|LPs?|AUM|family offices?|under management|fundrais\w*|pre-?seed|"
|
|
420
|
+
r"venture (?:capital|firms?|funds?))(?![\w-])", re.I)
|
|
421
|
+
CLAUSE_BREAK = _lazy(r"(?<=[.;:!?])\s+|\s+[—–-]\s+|—")
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def _amount(number: str, scale: str | None) -> float:
|
|
425
|
+
number = number.rstrip(".,")
|
|
426
|
+
if re.fullmatch(r"\d{1,3}(?:([.,])\d{3})(?:\1\d{3})*", number):
|
|
427
|
+
value = float(re.sub(r"[.,]", "", number))
|
|
428
|
+
else:
|
|
429
|
+
try:
|
|
430
|
+
value = float(number.replace(",", ""))
|
|
431
|
+
except ValueError:
|
|
432
|
+
return 0.0
|
|
433
|
+
return value * _SCALE.get((scale or "").lower(), 1)
|
|
434
|
+
|
|
435
|
+
|
|
436
|
+
def raise_amounts(segment: str):
|
|
437
|
+
"""Spans of currency amounts of at least RAISE_SIZED (the upper bound of a range counts)."""
|
|
438
|
+
for match in MONEY.finditer(segment):
|
|
439
|
+
a, a_scale, b, b_scale, c, c_scale = match.groups()
|
|
440
|
+
if c is not None:
|
|
441
|
+
value = _amount(c, c_scale)
|
|
442
|
+
else:
|
|
443
|
+
value = max(_amount(a, a_scale or b_scale), _amount(b, b_scale) if b else 0)
|
|
444
|
+
if value >= RAISE_SIZED:
|
|
445
|
+
yield match.start(), match.end()
|
|
446
|
+
|
|
447
|
+
|
|
448
|
+
def _within_words(segment: str, first: tuple[int, int], second: tuple[int, int], words: int) -> bool:
|
|
449
|
+
"""Whether at most ``words`` words stand between two spans of one segment."""
|
|
450
|
+
(a, b), (c, d) = sorted((first, second))
|
|
451
|
+
return c >= b and len(FEW_WORDS.findall(segment[b:c])) <= words
|
|
452
|
+
|
|
453
|
+
|
|
454
|
+
def _around(segment: str, first: tuple[int, int], second: tuple[int, int], words: int) -> str:
|
|
455
|
+
"""The text of two spans and up to ``words`` words on either side of them."""
|
|
456
|
+
start, end = min(first[0], second[0]), max(first[1], second[1])
|
|
457
|
+
before = [m.start() for m in FEW_WORDS.finditer(segment[:start])][-words:]
|
|
458
|
+
after = [m.end() for m in FEW_WORDS.finditer(segment, end)][:words]
|
|
459
|
+
return segment[before[0] if before else start:after[-1] if after else end]
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
def fundraising_rule(text, title="", summary=""):
|
|
463
|
+
"""valuation-or-dilution, fundraising-figures, or None; lines and table cells are separate segments."""
|
|
464
|
+
subject = "\n".join((title, summary, text))
|
|
465
|
+
if EQUITY_TERMS.search(subject) or SAFE_TERM.search(subject):
|
|
466
|
+
return "valuation-or-dilution"
|
|
467
|
+
figures = False
|
|
468
|
+
for segment in re.split(r"[\n|]", subject):
|
|
469
|
+
amounts = list(raise_amounts(segment))
|
|
470
|
+
if not amounts:
|
|
471
|
+
continue
|
|
472
|
+
spans = [m.span() for m in SAFE_WORD.finditer(segment)]
|
|
473
|
+
if any(_within_words(segment, money, safe, 3) for money in amounts for safe in spans):
|
|
474
|
+
return "valuation-or-dilution"
|
|
475
|
+
if any(_within_words(segment, money, m.span(), CONTEXT_WORDS)
|
|
476
|
+
and RAISE_CONTEXT.search(_around(segment, money, m.span(), CONTEXT_WORDS))
|
|
477
|
+
for money in amounts for m in PERCENT.finditer(segment)):
|
|
478
|
+
return "valuation-or-dilution"
|
|
479
|
+
if any(_within_words(segment, money, m.span(), 3) for money in amounts for m in FUNDRAISING_TERM.finditer(segment)):
|
|
480
|
+
figures = True
|
|
481
|
+
elif any(INVESTOR_WORDING.search(clause) and any(raise_amounts(clause))
|
|
482
|
+
for clause in CLAUSE_BREAK.split(segment)):
|
|
483
|
+
figures = True
|
|
484
|
+
return "fundraising-figures" if figures else None
|
|
485
|
+
|
|
486
|
+
|
|
487
|
+
# -- Restriction, private prose and transcripts, moved from experiments/egregore-mind/export.py at
|
|
488
|
+
# 45257e4b. names_people reads the exporter's ordinary words through strings_in; both are copied
|
|
489
|
+
# verbatim below, and test_held.py fails when they drift apart.
|
|
490
|
+
def strings_in(value) -> list[str]:
|
|
491
|
+
if value is None or isinstance(value, bool):
|
|
492
|
+
return []
|
|
493
|
+
if isinstance(value, str):
|
|
494
|
+
return [value]
|
|
495
|
+
if isinstance(value, (int, float)):
|
|
496
|
+
return [str(value)]
|
|
497
|
+
if isinstance(value, dict):
|
|
498
|
+
out = []
|
|
499
|
+
for key in sorted(value, key=str):
|
|
500
|
+
out.extend(strings_in(value[key]))
|
|
501
|
+
return out
|
|
502
|
+
if isinstance(value, (list, tuple)):
|
|
503
|
+
out = []
|
|
504
|
+
for item in value:
|
|
505
|
+
out.extend(strings_in(item))
|
|
506
|
+
return out
|
|
507
|
+
return [str(value)]
|
|
508
|
+
|
|
509
|
+
|
|
510
|
+
ENGLISH_STOP = set("""
|
|
511
|
+
a about above across actually after again against all almost along already also although always am among an and
|
|
512
|
+
another any anybody anyone anything anyway anywhere are aren around as ask asked asks at away back be became because
|
|
513
|
+
become becomes been before behind being below beside besides between beyond both but by came can cannot cant could
|
|
514
|
+
couldn did didn different do does doesn doing don done down during each either else elsewhere enough especially even
|
|
515
|
+
ever every everyone everything exactly except far few for former from further get gets getting give given gives
|
|
516
|
+
go goes going gone got gotten had hadn happen has hasn have haven having he hence her here hers herself him himself his
|
|
517
|
+
how however i ie if in inside instead into is isn it its itself just keep keeps kept know known knows last later least
|
|
518
|
+
less let lets like likely made make makes making many may maybe me meanwhile might mine more moreover most mostly much
|
|
519
|
+
must my myself near nearly need needed needs neither never nevertheless no nobody non none nor not nothing now nowhere
|
|
520
|
+
of off often oh ok okay on once one ones only onto or other others otherwise ought our ours ourselves out outside over
|
|
521
|
+
own per perhaps please put quite rather really said same say saying says see seem seemed seems seen several shall she
|
|
522
|
+
should shouldn since so some somebody somehow someone something sometime sometimes somewhat somewhere soon still such
|
|
523
|
+
sure take taken takes than that thats the their theirs them themselves then there thereby therefore these they thing
|
|
524
|
+
things this those though through throughout thus to together too took toward towards try trying under unless unlike
|
|
525
|
+
until up upon us use used useful uses using usually various very via was wasn way ways we well went were weren what
|
|
526
|
+
whatever when whenever where whereas wherever whether which while who whoever whole whom whose why will with within
|
|
527
|
+
without won would wouldn yes yet you your yours yourself yourselves etc e.g eg vs versus also within within upon
|
|
528
|
+
able across along among around beyond done each every here there want wants wanted lot lots bit kind sort really
|
|
529
|
+
""".split())
|
|
530
|
+
LABEL_GENERIC = set("""
|
|
531
|
+
decision decisions finding findings pattern patterns research document documents doc docs note notes summary
|
|
532
|
+
summaries context status update updates updated work works worked working team teams people person user users
|
|
533
|
+
system systems thing approach approaches question questions plan plans planning issue issues draft drafts drafted
|
|
534
|
+
version versions change changes changed review reviews reviewed idea ideas point points current currently next step
|
|
535
|
+
steps first second third new old good better best important key main core part parts level levels type types case
|
|
536
|
+
cases example examples problem problems time times day days week weeks weekly month months year years now value
|
|
537
|
+
values result results number numbers list lists item items section sections page pages file files line lines text
|
|
538
|
+
content contents thought thoughts reflection reflections finding wrap wraps open opened close closed closing start
|
|
539
|
+
started starting end ended ending done todo todos task tasks action actions owner owners author authors source
|
|
540
|
+
sources reference references link links title titles topic topics tag tags format formats template templates spec
|
|
541
|
+
specs feature features fix fixes fixed add added adds adding remove removed removes create created creates creating
|
|
542
|
+
build built builds building run runs running ran test tests tested testing check checks checked note noted output
|
|
543
|
+
outputs input inputs set sets setting settings support supports supported mode modes path paths name names named
|
|
544
|
+
call calls called flow flows state states based base layer layers default defaults option options process processes
|
|
545
|
+
direction directions specific general simple complex clear real actual full total single multiple across overall
|
|
546
|
+
model models framework frameworks structure structures thinking think thinks thought understanding understand
|
|
547
|
+
make makes making made right wrong high low long short small large big early late previous following follow follows
|
|
548
|
+
whole across inside outside question answer answers answered discussion discuss discussed meeting meetings call
|
|
549
|
+
proposal proposed propose proposes doc document captured capture captures record records recorded log logs logged
|
|
550
|
+
agent agents claude human humans conversation conversations repo repos repository commit commits branch branches
|
|
551
|
+
two three four five six seven eight nine ten related relevant ago min mins minute minutes hour hours sec secs
|
|
552
|
+
seconds org orgs organization organizations organizational knowledge project projects entry entries url urls dev
|
|
553
|
+
graph graphs neo neo4j cypher evidence surface surfaces command commands product products strategy strategies
|
|
554
|
+
shared individual within without another however across instead around including include includes included
|
|
555
|
+
specification asked ask asks session sessions handoff handoffs today yesterday active inactive pending complete
|
|
556
|
+
completed signal signals align aligned alignment framework approach mechanism mechanisms concept concepts
|
|
557
|
+
lead leads card cards brief briefs query queries tool tools read reads live gate gates mark marks remains remain
|
|
558
|
+
bin init bump zero view views activity quest quests code substrate data stack layer surface create stab ripcord
|
|
559
|
+
transcript transcripts capture captured attendant recorder flight dashboard session develop main skill skills
|
|
560
|
+
remember remembering writes write written image images displayed display original merged merging analyzing
|
|
561
|
+
cross chunk chunks rest persist persists persisted prime reading respect receive
|
|
562
|
+
share shared worth keeping works stated self improvement feel character application applications client clients
|
|
563
|
+
customer customers business offer offers enterprise vision tech intelligence architecture pipeline pipelines
|
|
564
|
+
""".split())
|
|
565
|
+
|
|
566
|
+
|
|
567
|
+
# Restrictions authors write. A restriction field that names specific people ("audience: Ada and
|
|
568
|
+
# Bo", "for: [ada]") or says private, confidential, founders or not for sharing holds the document;
|
|
569
|
+
# so does "not for sharing" or "do not share" in the title or the opening lines of the body.
|
|
570
|
+
# A document is never held merely for mentioning a person, and ``to``/``author`` are not restrictions.
|
|
571
|
+
RESTRICTION_KEYS = ("audience", "visibility", "share", "sharing", "shared_with", "for", "restricted",
|
|
572
|
+
"restricted_to", "access", "confidentiality", "distribution", "private", "confidential")
|
|
573
|
+
RESTRICTION_FLAGS = ("private", "confidential", "restricted")
|
|
574
|
+
RESTRICTED_MARKING = _lazy(
|
|
575
|
+
r"\b(?:private|confidential|founders?(?:[- ]only)?|restricted|secret|eyes[- ]only|personal|"
|
|
576
|
+
r"not (?:for|to be) (?:shared|sharing|distribution|distributed|circulation|circulated)|"
|
|
577
|
+
r"do not (?:share|distribute|circulate|forward))\b", re.I)
|
|
578
|
+
NOT_FOR_SHARING = _lazy(
|
|
579
|
+
r"\bnot (?:for|to be) (?:sharing|shared|distribution|distributed|circulation|circulated)\b|"
|
|
580
|
+
r"\b(?:do not|don['’]t|never) (?:share|distribute|circulate|forward)"
|
|
581
|
+
r"(?=\s*(?:[.!,;:)\]—–-]|$|(?:this|these|it|outside|beyond|with|externally|publicly|further|widely|any ?more)\b))",
|
|
582
|
+
re.I | re.M)
|
|
583
|
+
OPENING_LINES = 5
|
|
584
|
+
# A confidentiality banner (2026-10-12): a title or one of the opening body lines in which, once
|
|
585
|
+
# markdown emphasis is removed, one piece between separators is only the marking
|
|
586
|
+
# ("**February 2026 | Confidential**", "> INTERNAL ONLY", "Classification: Founders only").
|
|
587
|
+
# A marking inside a sentence ("a founder-only invitation stage", "high confidentiality
|
|
588
|
+
# requirements") is not a banner.
|
|
589
|
+
BANNER = _lazy(
|
|
590
|
+
r"(?:strictly |highly )?(?:private (?:and|&) )?confidential(?: (?:draft|document|memo|material))?|"
|
|
591
|
+
r"(?:for )?internal(?: use)? only|(?:co-?)?founders?['’]?(?: eyes)? only|(?:co-?)?founder[- ]only|"
|
|
592
|
+
r"not for (?:distribution|circulation|sharing)|do not (?:distribute|circulate|forward|share)|"
|
|
593
|
+
r"(?:private|restricted)(?: (?:and|&) confidential)?|eyes only", re.I)
|
|
594
|
+
BANNER_SPLIT = _lazy(r"\s*(?:[|·•:;,()\[\]]|\s[—–-]\s|—)\s*")
|
|
595
|
+
|
|
596
|
+
|
|
597
|
+
def banner_marking(line: str) -> bool:
|
|
598
|
+
plain_line = re.sub(r"[*_`#>~]+|^\s*[-+]\s+", " ", line or "")
|
|
599
|
+
plain_line = re.sub(r"[\U0001F300-\U0001FAFF☀-➿]", " ", plain_line)
|
|
600
|
+
return any(BANNER.fullmatch(" ".join(piece.split()).strip(" .!")) for piece in BANNER_SPLIT.split(plain_line))
|
|
601
|
+
|
|
602
|
+
|
|
603
|
+
# Words of a restriction value that describe a group or a use, not a person.
|
|
604
|
+
AUDIENCE_WORDS = set("""
|
|
605
|
+
org organization organisation orgs team teams all everyone everybody anyone member members internal public
|
|
606
|
+
company companies staff employee employees investor investors partner partners client clients customer customers
|
|
607
|
+
external grant grants application applications deck decks reference references comms communication communications
|
|
608
|
+
product products technical tech engineering engineers developers developer designers designer data room rooms
|
|
609
|
+
community communities user users reader readers general shared wide prep facing board advisors advisor
|
|
610
|
+
collaborators collaborator contributors contributor operators operator admins admin owners owner maintainers
|
|
611
|
+
maintainer leadership management ops sales marketing research researchers press media web site newsletter social
|
|
612
|
+
followers audience agents agent humans human people self myself future me us we you next session sessions
|
|
613
|
+
review reviewers reviewer legal finance funding funders stakeholders stakeholder ecosystem network friends
|
|
614
|
+
alpha beta testers tester pilot pilots partners prospects prospect lead leads leadership executive executives
|
|
615
|
+
founder founders cofounders public-facing investor-facing org-wide team-wide company-wide any none open closed
|
|
616
|
+
password gated encrypted link links the and for of to in on with only
|
|
617
|
+
""".split())
|
|
618
|
+
|
|
619
|
+
|
|
620
|
+
def _value_text(value) -> str:
|
|
621
|
+
return " ; ".join(strings_in(value)) if not isinstance(value, str) else value
|
|
622
|
+
|
|
623
|
+
|
|
624
|
+
def names_people(value) -> bool:
|
|
625
|
+
"""A restriction value that names a person: an entry of one or two words of two or more letters,
|
|
626
|
+
none of them a group, role, use or ordinary word and none an all-capitals acronym. Parenthesised
|
|
627
|
+
notes ("Ada (form review)") are not entries."""
|
|
628
|
+
text = re.sub(r"\([^)]*\)", " ", _value_text(value)).replace("[", " ").replace("]", " ")
|
|
629
|
+
ordinary = AUDIENCE_WORDS | ENGLISH_STOP | LABEL_GENERIC
|
|
630
|
+
for entry in re.split(r"[,;/+&|]|\band\b|\bor\b", text, flags=re.I):
|
|
631
|
+
words = re.findall(r"[^\W\d_]+", entry)
|
|
632
|
+
if not 1 <= len(words) <= 2 or len(entry.strip()) > 40:
|
|
633
|
+
continue
|
|
634
|
+
if any(len(w) < 2 or fold(w) in ordinary or w.isupper() for w in words):
|
|
635
|
+
continue
|
|
636
|
+
return True
|
|
637
|
+
return False
|
|
638
|
+
|
|
639
|
+
|
|
640
|
+
def restriction_rule(fm, title, body):
|
|
641
|
+
"""restricted-audience, restricted-marking, not-for-sharing, or None."""
|
|
642
|
+
for key in RESTRICTION_KEYS:
|
|
643
|
+
value = fm.get(key)
|
|
644
|
+
if value is None or value == "" or value == []:
|
|
645
|
+
continue
|
|
646
|
+
flag = value is True or str(value).strip().lower() in {"true", "yes"}
|
|
647
|
+
refused = value is False or str(value).strip().lower() in {"false", "no", "none"}
|
|
648
|
+
if (key in RESTRICTION_FLAGS and flag) or (key in {"share", "sharing"} and refused):
|
|
649
|
+
return "restricted-marking"
|
|
650
|
+
if isinstance(value, bool):
|
|
651
|
+
continue
|
|
652
|
+
if RESTRICTED_MARKING.search(_value_text(value)):
|
|
653
|
+
return "restricted-marking"
|
|
654
|
+
if key not in RESTRICTION_FLAGS and names_people(value):
|
|
655
|
+
return "restricted-audience"
|
|
656
|
+
lines = [line for line in (body or "").split("\n") if line.strip()][:OPENING_LINES]
|
|
657
|
+
opening = "\n".join(lines)
|
|
658
|
+
if NOT_FOR_SHARING.search(title or "") or NOT_FOR_SHARING.search(opening):
|
|
659
|
+
return "not-for-sharing"
|
|
660
|
+
raw_title = fm.get("title") if isinstance(fm.get("title"), str) else ""
|
|
661
|
+
if any(banner_marking(line) for line in [title or "", raw_title] + lines):
|
|
662
|
+
return "restricted-marking"
|
|
663
|
+
return None
|
|
664
|
+
|
|
665
|
+
|
|
666
|
+
# Explicit credential assignments disclose access, even when a document is otherwise an ordinary
|
|
667
|
+
# handoff. Match the assertion of a value, not discussion of password UX or redacted placeholders.
|
|
668
|
+
CREDENTIAL_LABEL = r"(?:password|passphrase|access[-_ ]code|unlock[-_ ]code|api[-_ ]key|access[-_ ]token|secret)"
|
|
669
|
+
CREDENTIAL_VALUE = r"(?:`[^`\n]+`|\"[^\"\n]+\"|'[^'\n]+'|[^\s`\"'()<>{}\[\]]+)"
|
|
670
|
+
CREDENTIAL_ASSIGNMENT = _lazy(
|
|
671
|
+
r"(?P<label>\b" + CREDENTIAL_LABEL + r"\b)[ \t]*(?P<connector>[:=]|\(|(?:is|was|to)\b|(?=[`\"']))"
|
|
672
|
+
r"[ \t]*(?P<value>" + CREDENTIAL_VALUE + r")|--password(?:=|[ \t]+)(?P<command>" + CREDENTIAL_VALUE + r")", re.I)
|
|
673
|
+
CREDENTIAL_NONVALUES = set("""
|
|
674
|
+
a an the this that your our their my new old another required optional never not unset empty missing
|
|
675
|
+
unchanged rotated hidden redacted removed omitted masked placeholder example encrypted hashed stored
|
|
676
|
+
safe secure private secret present available enabled disabled used set needed requested accepted denied
|
|
677
|
+
invalid incorrect valid correct supplied provided entered chosen generated kept saved loaded read
|
|
678
|
+
unlock open access protect allow enable improve strengthen change reset rotate recover authenticate
|
|
679
|
+
string true false null none undefined env environment process os ask ask-me known standard
|
|
680
|
+
""".split())
|
|
681
|
+
CONTRACT_LIFECYCLE = _lazy(
|
|
682
|
+
r"\bcontract\b[^.\n]{0,80}\b(?:expir(?:e[ds]?|ing|ation)|renew(?:al|ed|ing)?|continuation|extension|"
|
|
683
|
+
r"ends?\s+(?:in|on|next|this))\b", re.I)
|
|
684
|
+
PRIVATE_DESCRIPTOR = _lazy(
|
|
685
|
+
r"\bprivate\s+(?:[\w-]+\s+){0,5}(?:brief(?:ing)?|memo|document|report|paper|letter|invitation)\b|"
|
|
686
|
+
r"\bpersonal\s+(?:mentorship|mentoring|coaching|advisory)\s+(?:invitation|letter|message)\b|"
|
|
687
|
+
r"\b(?:liquidity|bridge[- ]financ\w*)\b[^.\n]{0,100}\b(?:brief(?:ing)?|memo|plan)\b"
|
|
688
|
+
r"[^.\n]{0,60}\bfor\s+(?-i:[A-Z][a-z]+)\b", re.I)
|
|
689
|
+
|
|
690
|
+
|
|
691
|
+
def private_prose_rule(text, title="", summary="", fm=None):
|
|
692
|
+
"""Hold explicit credentials, personal captures and private-purpose prose for non-admins."""
|
|
693
|
+
if str((fm or {}).get("capture_mode") or "").strip().lower() == "personal":
|
|
694
|
+
return "personal-capture"
|
|
695
|
+
# Read source text, which already includes frontmatter summaries. A derived summary can drop
|
|
696
|
+
# angle-bracket placeholders and line breaks, making the next field look like a secret value.
|
|
697
|
+
subject = "\n".join((title or "", text or summary or ""))
|
|
698
|
+
for match in CREDENTIAL_ASSIGNMENT.finditer(subject):
|
|
699
|
+
raw = match.group("value") or match.group("command") or ""
|
|
700
|
+
quoted = raw.startswith(("`", '"', "'"))
|
|
701
|
+
value = raw.strip("`\"'").rstrip(".,;:!?")
|
|
702
|
+
token = value.lower()
|
|
703
|
+
label = (match.group("label") or "password").lower().replace("_", " ").replace("-", " ")
|
|
704
|
+
connector = match.group("connector") or "="
|
|
705
|
+
# Unquoted natural-language assignments can contain several words. An article
|
|
706
|
+
# alone is not enough to decide that the complete value is harmless.
|
|
707
|
+
clause = re.split(r"[\n.!?;]", subject[match.start("value"):], maxsplit=1)[0].strip() if match.group("value") else value
|
|
708
|
+
if not quoted and connector.lower() in {"is", "was", "to"}:
|
|
709
|
+
descriptive = re.match(r"(?:never|not)\b|(?:required|optional|active|inactive|unset|empty|missing|"
|
|
710
|
+
r"unchanged|rotated|hidden|redacted|removed|omitted|masked|encrypted|hashed|"
|
|
711
|
+
r"stored|available|enabled|disabled|invalid|incorrect|valid|correct|safe|secure)\b", clause, re.I)
|
|
712
|
+
if descriptive:
|
|
713
|
+
continue
|
|
714
|
+
if re.match(r"(?:a|an|the)\s+\S+", clause, re.I):
|
|
715
|
+
return "access-credential"
|
|
716
|
+
if token == "secret":
|
|
717
|
+
return "access-credential"
|
|
718
|
+
if label not in {"password", "passphrase", "access code", "unlock code"} and not quoted and connector == "(":
|
|
719
|
+
continue # A type annotation such as "API key (JWT)" asserts no value.
|
|
720
|
+
if connector == "(" and not quoted and not subject[match.end():].lstrip().startswith(")"):
|
|
721
|
+
continue # A parenthesised instruction such as "password (ask me privately)".
|
|
722
|
+
if (value and re.search(r"[a-z0-9]", token)
|
|
723
|
+
and (quoted or connector in {":", "="} or token not in CREDENTIAL_NONVALUES)
|
|
724
|
+
and not token.startswith(("[", "{", "$", "process.", "os.", "env.", "secrets."))):
|
|
725
|
+
return "access-credential"
|
|
726
|
+
if CONTRACT_LIFECYCLE.search(subject):
|
|
727
|
+
return "contract-lifecycle"
|
|
728
|
+
if PRIVATE_DESCRIPTOR.search(subject):
|
|
729
|
+
return "private-purpose"
|
|
730
|
+
return None
|
|
731
|
+
|
|
732
|
+
|
|
733
|
+
TRANSCRIPT_NAME = _lazy(r"[-_](?:raw|raw[-_]transcript|transcript[-_]raw)$", re.I)
|
|
734
|
+
TRANSCRIPT_TYPE_KEYS = ("record_type", "type", "kind")
|
|
735
|
+
# Frontmatter a raw capture carries beside a plain "transcript" type: its hash, speaker map or
|
|
736
|
+
# preservation note.
|
|
737
|
+
RAW_MARKERS = ("raw", "raw_sha256", "raw_hash", "speaker_mapping", "preservation")
|
|
738
|
+
|
|
739
|
+
|
|
740
|
+
def transcript_rule(rel, fm=None):
|
|
741
|
+
"""Raw meeting transcripts, whatever else their frontmatter says (2026-10-12): files under a
|
|
742
|
+
meetings/transcripts/ folder, files whose names end in -raw, -raw-transcript or -transcript-raw,
|
|
743
|
+
and frontmatter whose record_type, type or kind is a raw transcript ("raw_transcript",
|
|
744
|
+
"raw-transcript", "transcript-raw") or "transcript" beside a raw marker. Checked after the
|
|
745
|
+
personnel rule, so documents that rule already held keep their recorded rule."""
|
|
746
|
+
folders = [part.lower() for part in Path(rel.removeprefix("memory/")).parts[:-1]]
|
|
747
|
+
if (any(a == "meetings" and b == "transcripts" for a, b in zip(folders, folders[1:]))
|
|
748
|
+
or TRANSCRIPT_NAME.search(Path(rel).stem)):
|
|
749
|
+
return "raw-transcript"
|
|
750
|
+
fm = fm or {}
|
|
751
|
+
for key in TRANSCRIPT_TYPE_KEYS:
|
|
752
|
+
words = set(re.findall(r"[a-z]+", str(fm.get(key) or "").lower()))
|
|
753
|
+
if "transcript" in words and ("raw" in words or (words == {"transcript"} and any(
|
|
754
|
+
fm.get(marker) not in (None, "", [], False) for marker in RAW_MARKERS))):
|
|
755
|
+
return "raw-transcript"
|
|
756
|
+
return None
|
|
757
|
+
|
|
758
|
+
|
|
759
|
+
# -- The exporter's name strippers, moved from experiments/egregore-mind/export.py (H3) ---------
|
|
760
|
+
# Summaries shown to anyone but admins quote no memory paths at all: a quoted path can name a
|
|
761
|
+
# document that audience does not see (admin-only, held, captured or private), and a path tells a
|
|
762
|
+
# reader nothing the node's own place on the map does not.
|
|
763
|
+
MEMORY_PATH = _lazy(r"(?<![\w.-])memory/[^\s`'\")\]>,;]*")
|
|
764
|
+
|
|
765
|
+
|
|
766
|
+
def strip_paths(text: str, pattern: re.Pattern = MEMORY_PATH) -> str:
|
|
767
|
+
"""Remove every match of ``pattern`` and the empty quotes, brackets and separators it leaves."""
|
|
768
|
+
if not pattern.search(text):
|
|
769
|
+
return text
|
|
770
|
+
text = pattern.sub(lambda m: "…" if m.group().endswith("…") else "", text)
|
|
771
|
+
text = re.sub(r"`\s*`|\(\s*[,;\s]*\)|\[\s*[,;\s]*\]", "", text)
|
|
772
|
+
text = re.sub(r"\s*[,;](?:\s*[,;])+", ",", text)
|
|
773
|
+
text = re.sub(r"\s+([,.;:!?…])", r"\1", text)
|
|
774
|
+
return " ".join(text.split()).strip(" ,;")
|
|
775
|
+
|
|
776
|
+
|
|
777
|
+
def strip_memory_paths(text: str) -> str:
|
|
778
|
+
return strip_paths(text, MEMORY_PATH)
|
|
779
|
+
|
|
780
|
+
|
|
781
|
+
def file_name_pattern(names, slugs=()) -> re.Pattern | None:
|
|
782
|
+
"""A withheld document's file name, bare or after any path token ("x.md", "a//b/x.md",
|
|
783
|
+
"~user/x.md", "file:///a/x.md", "https://host/a/x.md"), and its slug without ".md" ("a-b",
|
|
784
|
+
"notes/a-b") when the slug is distinctive (``distinctive_slug``). ``names_in`` and
|
|
785
|
+
``strip_names`` check the percent-decoded text as well."""
|
|
786
|
+
names = {n for n in names if n}
|
|
787
|
+
slugs = {s for s in slugs if s}
|
|
788
|
+
options = [(n, re.escape(n)) for n in names] + [(s, re.escape(s) + r"(?:\.(?:md|html?))?") for s in slugs]
|
|
789
|
+
if not options:
|
|
790
|
+
return None
|
|
791
|
+
options.sort(key=lambda option: (-len(option[0]), option[0]))
|
|
792
|
+
return re.compile(r"(?<![\w.-])(?:[^\s`'\"()\[\]<>]*/)?(?:" + "|".join(p for _, p in options) + r")(?![\w/-])",
|
|
793
|
+
re.IGNORECASE)
|
|
794
|
+
|
|
795
|
+
|
|
796
|
+
def spellings(text: str) -> list[str]:
|
|
797
|
+
"""The text, then percent- and entity-decoded until it stops changing ("forest%252D3.md",
|
|
798
|
+
"forest-3.md"), at most three passes."""
|
|
799
|
+
found = [text]
|
|
800
|
+
for _ in range(3):
|
|
801
|
+
decoded = html.unescape(unquote(found[-1]))
|
|
802
|
+
if decoded == found[-1]:
|
|
803
|
+
break
|
|
804
|
+
found.append(decoded)
|
|
805
|
+
return found
|
|
806
|
+
|
|
807
|
+
|
|
808
|
+
def names_in(pattern: re.Pattern | None, text: str) -> bool:
|
|
809
|
+
"""A withheld name in text, as written or decoded, in any letter case ("Forest-3.md")."""
|
|
810
|
+
return bool(pattern and any(pattern.search(spelling) for spelling in spellings(text)))
|
|
811
|
+
|
|
812
|
+
|
|
813
|
+
def strip_names(text: str, pattern: re.Pattern) -> str:
|
|
814
|
+
"""``strip_paths`` for withheld names, decoded spellings included."""
|
|
815
|
+
text = strip_paths(text, pattern)
|
|
816
|
+
decoded = spellings(text)
|
|
817
|
+
return strip_paths(decoded[-1], pattern) if any(pattern.search(d) for d in decoded[1:]) else text
|
|
818
|
+
|
|
819
|
+
|
|
820
|
+
def qualified_names(outside, shown) -> set[str]:
|
|
821
|
+
"""For an outside path whose file name a shown document shares, its shortest folder-qualified
|
|
822
|
+
spelling that no shown path ends with ("findings/x.md" beside a shown "findings/ocean/x.md")."""
|
|
823
|
+
shown = [path.removeprefix("memory/") for path in shown]
|
|
824
|
+
names, found = {path.rsplit("/", 1)[-1] for path in shown}, set()
|
|
825
|
+
for parts in (path.removeprefix("memory/").split("/") for path in outside):
|
|
826
|
+
tails = ["/".join(parts[-k:]) for k in range(2, len(parts) + 1)] if parts[-1] in names else []
|
|
827
|
+
found.update(next(([t] for t in tails if not any(p == t or p.endswith("/" + t) for p in shown)), []))
|
|
828
|
+
return found
|
|
829
|
+
|
|
830
|
+
|
|
831
|
+
# -- A document as the exporter reads it -------------------------------------------------------
|
|
832
|
+
# Copied verbatim from experiments/egregore-mind/export.py (read_frontmatter, title_for, summary_for
|
|
833
|
+
# and their helpers), so a verdict here reads exactly the exporter's words; test_held.py fails when
|
|
834
|
+
# the two readings drift apart.
|
|
835
|
+
ROLE_MASK = _lazy(r"\[(?:person|organization)\]", re.I) # mind/organize.py
|
|
836
|
+
|
|
837
|
+
|
|
838
|
+
SUMMARY_KEYS = ("summary", "description", "why_it_matters")
|
|
839
|
+
EMAIL = _lazy(r"(?<![\w.+-])[A-Za-z0-9._%+-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)*\.[A-Za-z]{2,}")
|
|
840
|
+
WEB_LINK = _lazy(r"(?:\b(?:https?|ftp)://|\bwww\.)([^\s/?#)\]>\"'`]+)[^\s)\]>\"'`]*", re.I)
|
|
841
|
+
LONG_ID = _lazy(r"\b([0-9a-f]{8})(?:-[0-9a-f]{4}){3}-[0-9a-f]{12}\b|\b([0-9a-f]{8})[0-9a-f]{24,}\b", re.I)
|
|
842
|
+
LOCAL_PATH = _lazy(r"(?:file://)?(?<![\w.~])(?:/Users|/home|/private|/tmp|/var/folders|/Volumes)/[^\s)\]>\"'`,;]*")
|
|
843
|
+
SLUG_TITLE = _lazy(r"^(?:([A-Z][\w ]{0,24}):\s+)?([a-z0-9çğıöşüéèáíóúñäë ]+)$")
|
|
844
|
+
TLDS = {"com", "io", "ai", "so", "co", "dev", "app", "org", "net", "xyz", "tech", "eu", "de", "tr", "me", "sh",
|
|
845
|
+
"gg", "ly", "us", "uk", "fm", "tv", "to", "cc", "info", "edu", "gov"}
|
|
846
|
+
SLUG_PATH_WORDS = {"users", "home", "private", "tmp", "var", "library", "downloads", "desktop", "documents", "file"}
|
|
847
|
+
FENCE = _lazy(r"```.*?(?:```|\Z)|~~~.*?(?:~~~|\Z)", re.S)
|
|
848
|
+
HTML = _lazy(r"<!--.*?-->|<[^>\n]{1,200}>", re.S)
|
|
849
|
+
MDIMAGE = _lazy(r"!\[[^\]]*\]\([^)]*\)")
|
|
850
|
+
MDLINK = _lazy(r"\[([^\]]*)\]\([^)]*\)")
|
|
851
|
+
WIKILINK = _lazy(r"\[\[([^\]|#]+)(?:#[^\]|]*)?(?:\|([^\]]*))?\]\]")
|
|
852
|
+
EMPHASIS = _lazy(r"(\*\*|__|\*|(?<!\w)_|`+|~~)")
|
|
853
|
+
FRONTMATTER_LINE = _lazy(r"^([A-Za-z][A-Za-z0-9_-]*):(?:[ \t]*(.*))?$")
|
|
854
|
+
|
|
855
|
+
|
|
856
|
+
def plain(text: str) -> str:
|
|
857
|
+
"""Markdown/HTML -> one line of plain text."""
|
|
858
|
+
text = MDIMAGE.sub("", text)
|
|
859
|
+
text = WIKILINK.sub(lambda m: (m.group(2) or m.group(1)).strip(), text)
|
|
860
|
+
text = MDLINK.sub(r"\1", text)
|
|
861
|
+
text = HTML.sub(" ", text)
|
|
862
|
+
text = re.sub(r"^\s{0,3}(?:#{1,6}\s+|>\s?|[-*+]\s+|\d+[.)]\s+)", "", text, flags=re.M)
|
|
863
|
+
text = EMPHASIS.sub("", text)
|
|
864
|
+
return " ".join(text.split())
|
|
865
|
+
|
|
866
|
+
|
|
867
|
+
def clip(text: str, limit: int) -> str:
|
|
868
|
+
text = " ".join(text.split())
|
|
869
|
+
if len(text) <= limit:
|
|
870
|
+
return text
|
|
871
|
+
cut = text[: limit - 1]
|
|
872
|
+
space = cut.rfind(" ")
|
|
873
|
+
if space > limit * 0.6:
|
|
874
|
+
cut = cut[:space]
|
|
875
|
+
return cut.rstrip(" ,;:.-—–") + "…"
|
|
876
|
+
|
|
877
|
+
|
|
878
|
+
def _host(match: re.Match) -> str:
|
|
879
|
+
host = match.group(1).rsplit("@", 1)[-1].split(":", 1)[0].lower().rstrip(".,;:!?")
|
|
880
|
+
return host.removeprefix("www.")
|
|
881
|
+
|
|
882
|
+
|
|
883
|
+
def _last_part(match: re.Match) -> str:
|
|
884
|
+
parts = [p for p in match.group(0).rstrip(".,;:!?").split("/") if p]
|
|
885
|
+
last = parts[-1] if parts else ""
|
|
886
|
+
return "" if _id_like(last) else last
|
|
887
|
+
|
|
888
|
+
|
|
889
|
+
def scrub(text: str) -> str:
|
|
890
|
+
"""Emails -> [email], links -> their host, local paths -> their last part, UUIDs and long
|
|
891
|
+
hashes -> their first 8 characters (an emissary or share id works like the link itself)."""
|
|
892
|
+
text = ROLE_MASK.sub("", text)
|
|
893
|
+
text = EMAIL.sub("[email]", text)
|
|
894
|
+
text = WEB_LINK.sub(_host, text)
|
|
895
|
+
text = LOCAL_PATH.sub(_last_part, text)
|
|
896
|
+
text = LONG_ID.sub(lambda m: m.group(1) or m.group(2), text)
|
|
897
|
+
return " ".join(text.split())
|
|
898
|
+
|
|
899
|
+
|
|
900
|
+
def _id_like(word: str) -> bool:
|
|
901
|
+
"""Share ids, hashes, tracking parameters; not 'v2', 'b2b', 'pg16', 'gpt4o' or 'qwen3'."""
|
|
902
|
+
digits = sum(ch.isdigit() for ch in word)
|
|
903
|
+
if not digits:
|
|
904
|
+
return False
|
|
905
|
+
if digits == len(word):
|
|
906
|
+
return len(word) >= 5
|
|
907
|
+
hexish = all(ch in "0123456789abcdefABCDEF" for ch in word)
|
|
908
|
+
turns = sum(1 for a, b in zip(word, word[1:]) if a.isdigit() != b.isdigit())
|
|
909
|
+
return (hexish and len(word) >= 4) or turns >= 3 or (turns >= 2 and len(word) >= 8) or len(word) >= 12
|
|
910
|
+
|
|
911
|
+
|
|
912
|
+
def tidy_slug(text: str) -> str:
|
|
913
|
+
"""A title made from a slugged prompt ('https chatgpt com s t 6a6e…', 'private tmp claude 501
|
|
914
|
+
users sample …'): hosts collapse to 'chatgpt.com'; ids, local folders and user names drop out."""
|
|
915
|
+
words = text.split()
|
|
916
|
+
pathy = "tmp" in words or sum(w in SLUG_PATH_WORDS for w in words) >= 2
|
|
917
|
+
out: list[str] = []
|
|
918
|
+
i = 0
|
|
919
|
+
while i < len(words):
|
|
920
|
+
w = words[i]
|
|
921
|
+
if w in ("https", "http", "ftp", "www"):
|
|
922
|
+
j = i + 1
|
|
923
|
+
while j < len(words) and words[j] == "www":
|
|
924
|
+
j += 1
|
|
925
|
+
host: list[str] = []
|
|
926
|
+
while j < len(words) and len(host) < 4:
|
|
927
|
+
host.append(words[j])
|
|
928
|
+
j += 1
|
|
929
|
+
if host[-1] in TLDS and len(host) >= 2:
|
|
930
|
+
break
|
|
931
|
+
if len(host) >= 2 and host[-1] in TLDS:
|
|
932
|
+
out.append(".".join(host))
|
|
933
|
+
i = j
|
|
934
|
+
while i < len(words) and (_id_like(words[i]) or len(words[i]) == 1):
|
|
935
|
+
i += 1 # the path right after a host: share ids, single-letter route parts
|
|
936
|
+
continue
|
|
937
|
+
i += 1
|
|
938
|
+
continue
|
|
939
|
+
if w in ("users", "home") and i + 1 < len(words) and (
|
|
940
|
+
pathy or (i + 2 < len(words) and words[i + 2] in SLUG_PATH_WORDS | {"claude", "dev", "code"})):
|
|
941
|
+
i += 2 # a local user folder and the user name after it
|
|
942
|
+
continue
|
|
943
|
+
if pathy and w in ("private", "tmp", "var", "file"):
|
|
944
|
+
i += 1
|
|
945
|
+
continue
|
|
946
|
+
if pathy and w == "claude" and i + 1 < len(words) and words[i + 1].isdigit():
|
|
947
|
+
i += 2
|
|
948
|
+
continue
|
|
949
|
+
if not _id_like(w):
|
|
950
|
+
out.append(w)
|
|
951
|
+
i += 1
|
|
952
|
+
return " ".join(out)
|
|
953
|
+
|
|
954
|
+
|
|
955
|
+
def _scalar(raw: str):
|
|
956
|
+
value = raw.strip()
|
|
957
|
+
if not value:
|
|
958
|
+
return None
|
|
959
|
+
try:
|
|
960
|
+
return json.loads(value)
|
|
961
|
+
except (json.JSONDecodeError, ValueError):
|
|
962
|
+
pass
|
|
963
|
+
if value.startswith("[") and value.endswith("]"):
|
|
964
|
+
inner = value[1:-1].strip()
|
|
965
|
+
return [p.strip().strip("\"'") for p in inner.split(",") if p.strip()] if inner else []
|
|
966
|
+
if len(value) >= 2 and value[0] == value[-1] and value[0] in "'\"":
|
|
967
|
+
return value[1:-1]
|
|
968
|
+
low = value.lower()
|
|
969
|
+
if low in ("true", "false"):
|
|
970
|
+
return low == "true"
|
|
971
|
+
if low in ("null", "~"):
|
|
972
|
+
return None
|
|
973
|
+
return value
|
|
974
|
+
|
|
975
|
+
|
|
976
|
+
def _header(normalized: str) -> str | None:
|
|
977
|
+
if not normalized.startswith("---\n"):
|
|
978
|
+
return None
|
|
979
|
+
closing = normalized.find("\n---\n", 4)
|
|
980
|
+
if closing < 0:
|
|
981
|
+
closing = normalized.find("\n---", 4)
|
|
982
|
+
if closing < 0 or normalized[closing + 4:].strip():
|
|
983
|
+
return None
|
|
984
|
+
return normalized[4:closing]
|
|
985
|
+
|
|
986
|
+
|
|
987
|
+
def _block_lists(header: str, fields: dict) -> dict:
|
|
988
|
+
"""Recover YAML block lists (``key:`` followed by `` - item``) the strict reader skips."""
|
|
989
|
+
current = None
|
|
990
|
+
items: dict[str, list] = {}
|
|
991
|
+
for line in header.splitlines():
|
|
992
|
+
if not line.strip():
|
|
993
|
+
continue
|
|
994
|
+
if not line[:1].isspace():
|
|
995
|
+
match = FRONTMATTER_LINE.match(line)
|
|
996
|
+
current = match.group(1) if match and not (match.group(2) or "").strip() else None
|
|
997
|
+
continue
|
|
998
|
+
if current and fields.get(current) is None:
|
|
999
|
+
item = re.match(r"^\s+-\s+(.*)$", line)
|
|
1000
|
+
if item:
|
|
1001
|
+
items.setdefault(current, []).append(_scalar(item.group(1)))
|
|
1002
|
+
if items:
|
|
1003
|
+
fields = dict(fields)
|
|
1004
|
+
fields.update(items)
|
|
1005
|
+
return fields
|
|
1006
|
+
|
|
1007
|
+
|
|
1008
|
+
def read_frontmatter(text: str, split_frontmatter) -> tuple[dict, str, bool]:
|
|
1009
|
+
"""Returns (fields, body, lenient). Strict reader first; tolerant fallback."""
|
|
1010
|
+
normalized = text.replace("\r\n", "\n").replace("\r", "\n")
|
|
1011
|
+
try:
|
|
1012
|
+
fields, body = split_frontmatter(normalized)
|
|
1013
|
+
if fields and any(v is None for v in fields.values()):
|
|
1014
|
+
header = _header(normalized)
|
|
1015
|
+
if header is not None:
|
|
1016
|
+
fields = _block_lists(header, fields)
|
|
1017
|
+
return fields, body, False
|
|
1018
|
+
except Exception: # noqa: BLE001 - legacy files: parse what we can
|
|
1019
|
+
header = _header(normalized)
|
|
1020
|
+
if header is None:
|
|
1021
|
+
return {}, normalized, True
|
|
1022
|
+
fields: dict = {}
|
|
1023
|
+
for line in header.splitlines():
|
|
1024
|
+
if not line.strip() or line[:1].isspace() or line.lstrip().startswith("#"):
|
|
1025
|
+
continue
|
|
1026
|
+
match = FRONTMATTER_LINE.match(line)
|
|
1027
|
+
if match and match.group(1) not in fields:
|
|
1028
|
+
fields[match.group(1)] = _scalar(match.group(2) or "")
|
|
1029
|
+
fields = _block_lists(header, fields)
|
|
1030
|
+
end = normalized.find("\n---", 4)
|
|
1031
|
+
body = normalized[end + 4:].lstrip("-").lstrip("\n") if end >= 0 else normalized
|
|
1032
|
+
return fields, body, True
|
|
1033
|
+
|
|
1034
|
+
|
|
1035
|
+
def title_for(fm: dict, body: str, rel: str) -> str:
|
|
1036
|
+
raw = fm.get("title")
|
|
1037
|
+
title = plain(str(raw)) if raw not in (None, "") and not isinstance(raw, (list, dict)) else ""
|
|
1038
|
+
if not title:
|
|
1039
|
+
heading = re.search(r"(?m)^#\s+(.+?)\s*#*\s*$", body)
|
|
1040
|
+
if heading:
|
|
1041
|
+
title = plain(heading.group(1))
|
|
1042
|
+
if not title:
|
|
1043
|
+
stem = rel.rsplit("/", 1)[-1][:-3]
|
|
1044
|
+
stem = re.sub(r"^(?:20\d{2}-\d{2}-\d{2}|\d{2})[-_]?", "", stem) or stem
|
|
1045
|
+
title = " ".join(re.split(r"[-_]+", stem)).strip()
|
|
1046
|
+
title = title[:1].upper() + title[1:]
|
|
1047
|
+
title = scrub(title)
|
|
1048
|
+
slug = SLUG_TITLE.match(title)
|
|
1049
|
+
if slug and len(slug.group(2).split()) >= 3:
|
|
1050
|
+
prefix, rest = slug.group(1), tidy_slug(slug.group(2))
|
|
1051
|
+
if prefix: # "Ripcord: <slugged prompt>": keep the kind alone when little survives
|
|
1052
|
+
title = f"{prefix}: {rest}" if len(re.findall(r"[^\W\d_]{3,}", rest)) >= 2 else prefix
|
|
1053
|
+
elif rest:
|
|
1054
|
+
title = rest
|
|
1055
|
+
return clip(title, 120)
|
|
1056
|
+
|
|
1057
|
+
|
|
1058
|
+
META_LINE = _lazy(r"^\s*(?:[-*]\s+)?\**[A-Za-z][\w /&'-]{0,32}\**\s*:\**\s")
|
|
1059
|
+
META_KEY = _lazy(r"^\s*(?:[-*]\s+)?\**([A-Za-z][\w /&'-]{0,32}?)\**\s*:\**\s*(.*)$")
|
|
1060
|
+
META_NOISE = {"date", "author", "authors", "status", "type", "topics", "tags", "created", "updated", "from", "to",
|
|
1061
|
+
"owner", "id", "version", "source", "url", "link", "category", "project", "quest", "quests",
|
|
1062
|
+
"urgency", "confidence", "conviction", "time", "duration", "attendees", "participants", "branch",
|
|
1063
|
+
"repo", "pr", "base", "session", "started", "ended", "recipient", "addressed to", "kind", "priority"}
|
|
1064
|
+
|
|
1065
|
+
|
|
1066
|
+
def summary_for(fm: dict, body: str) -> str:
|
|
1067
|
+
fm_fallback = ""
|
|
1068
|
+
for key in SUMMARY_KEYS:
|
|
1069
|
+
value = fm.get(key)
|
|
1070
|
+
if isinstance(value, str) and value.strip():
|
|
1071
|
+
text = scrub(plain(value))
|
|
1072
|
+
if len(text) >= 20:
|
|
1073
|
+
return clip(text, 220)
|
|
1074
|
+
fm_fallback = fm_fallback or text
|
|
1075
|
+
text = FENCE.sub("\n\n", body)
|
|
1076
|
+
fallback = ""
|
|
1077
|
+
meta_fallback = ""
|
|
1078
|
+
for block in re.split(r"\n\s*\n", text):
|
|
1079
|
+
lines = [line for line in block.strip().splitlines() if line.strip()]
|
|
1080
|
+
if not lines:
|
|
1081
|
+
continue
|
|
1082
|
+
head = lines[0].lstrip()
|
|
1083
|
+
if head.startswith(("#", "|", "---", "***", "<!--", "![", "```", "~~~", "[//]")):
|
|
1084
|
+
continue
|
|
1085
|
+
if all(META_LINE.match(line) for line in lines):
|
|
1086
|
+
if not meta_fallback:
|
|
1087
|
+
parts = []
|
|
1088
|
+
for line in lines:
|
|
1089
|
+
match = META_KEY.match(line)
|
|
1090
|
+
if match and match.group(1).strip().lower() not in META_NOISE and match.group(2).strip():
|
|
1091
|
+
parts.append(f"{match.group(1).strip()}: {match.group(2).strip()}")
|
|
1092
|
+
meta_fallback = scrub(plain(" · ".join(parts)))
|
|
1093
|
+
continue
|
|
1094
|
+
if head.startswith(("- ", "* ", "+ ")) or re.match(r"^\d+[.)]\s", head):
|
|
1095
|
+
if not fallback:
|
|
1096
|
+
fallback = scrub(plain(" ".join(lines)))
|
|
1097
|
+
continue
|
|
1098
|
+
candidate = scrub(plain(" ".join(lines)))
|
|
1099
|
+
if len(candidate) >= 40:
|
|
1100
|
+
return clip(candidate, 220)
|
|
1101
|
+
if not fallback and candidate:
|
|
1102
|
+
fallback = candidate
|
|
1103
|
+
return clip(fallback or meta_fallback or fm_fallback, 220)
|
|
1104
|
+
|
|
1105
|
+
|
|
1106
|
+
def document_rule(rel: str, text: str, fm: dict, body: str, title: str, summary: str, after: int = 1) -> str | None:
|
|
1107
|
+
"""The first rule past class ``after`` that holds a parsed document, in the exporter's order."""
|
|
1108
|
+
_work(text)
|
|
1109
|
+
checks = ((2, lambda: paperwork_rule({**fm, "title": title}, rel)), (3, lambda: personnel_rule(text, title, summary, rel)),
|
|
1110
|
+
(4, lambda: transcript_rule(rel, fm)), (5, lambda: fundraising_rule(text, title, summary)),
|
|
1111
|
+
(6, lambda: restriction_rule(fm, title, body)), (7, lambda: private_prose_rule(text, title, summary, fm)),
|
|
1112
|
+
(8, lambda: personnel_change_rule(text, title, summary, fm)))
|
|
1113
|
+
return next((name for number, check in checks if number > after and (name := check())), None)
|
|
1114
|
+
|
|
1115
|
+
|
|
1116
|
+
def _parse(rel: str, text: str, summary: bool = True) -> dict:
|
|
1117
|
+
"""A document as the exporter's members audience reads it (the summary only when asked)."""
|
|
1118
|
+
_work(text)
|
|
1119
|
+
fm, body, _lenient = read_frontmatter(text, split_frontmatter)
|
|
1120
|
+
return {"rel": rel, "fm": fm, "body": body, "title": title_for(fm, body, rel),
|
|
1121
|
+
"summary": strip_memory_paths(summary_for(fm, body)) if summary else None}
|
|
1122
|
+
|
|
1123
|
+
|
|
1124
|
+
def _rule(rel: str, text: str) -> str | None:
|
|
1125
|
+
"""The first rule that holds the document at memory path ``rel``."""
|
|
1126
|
+
doc = _parse(rel, text)
|
|
1127
|
+
return document_rule(rel, text, doc["fm"], doc["body"], doc["title"], doc["summary"])
|
|
1128
|
+
|
|
1129
|
+
|
|
1130
|
+
def classify(rel: str, text: str) -> str | None:
|
|
1131
|
+
"""``admin`` (class 1), a rule name, or None for a memory-relative path and its text."""
|
|
1132
|
+
return "admin" if document_is_admin(text) else _rule(rel, text)
|
|
1133
|
+
|
|
1134
|
+
|
|
1135
|
+
# -- The admin derivatives, moved from experiments/egregore-mind/export.py at 45257e4b ----------
|
|
1136
|
+
def distinctive_slug(stem: str) -> bool:
|
|
1137
|
+
"""A file slug that names one document rather than an ordinary word: two or more hyphenated
|
|
1138
|
+
parts, at least eight characters and at least one part of three or more letters."""
|
|
1139
|
+
parts = stem.split("-")
|
|
1140
|
+
return len(parts) >= 2 and len(stem) >= 8 and all(parts) and any(sum(c.isalpha() for c in p) >= 3 for p in parts)
|
|
1141
|
+
|
|
1142
|
+
|
|
1143
|
+
# A document whose text is wholly or almost wholly inside one admin-only document is a copy of it:
|
|
1144
|
+
# at least COPY_SHARE of its word COPY_WORDS-grams appear in that one document. Very short documents
|
|
1145
|
+
# (fewer than COPY_MIN_SHINGLES grams) are left to the other rules.
|
|
1146
|
+
COPY_RULE = "copy-of-admin-document"
|
|
1147
|
+
COPY_WORDS = 8
|
|
1148
|
+
COPY_SHARE = 0.9
|
|
1149
|
+
COPY_MIN_SHINGLES = 25
|
|
1150
|
+
|
|
1151
|
+
|
|
1152
|
+
def shingles(text: str) -> set[int]:
|
|
1153
|
+
_work(text)
|
|
1154
|
+
words = re.findall(r"[^\W_]+", fold(text or ""))
|
|
1155
|
+
if _WORK_BUDGET.get() is None:
|
|
1156
|
+
return {hash(" ".join(words[i:i + COPY_WORDS])) for i in range(len(words) - COPY_WORDS + 1)}
|
|
1157
|
+
result = set()
|
|
1158
|
+
for i in range(len(words) - COPY_WORDS + 1):
|
|
1159
|
+
_work(COPY_WORDS)
|
|
1160
|
+
result.add(hash(" ".join(words[i:i + COPY_WORDS])))
|
|
1161
|
+
return result
|
|
1162
|
+
|
|
1163
|
+
|
|
1164
|
+
# Records about an admin-only document (2026-10-12, third audit), checked with the copy rule after
|
|
1165
|
+
# every document is classified:
|
|
1166
|
+
# page-of-admin-document: the record is a hosted page or a derivative of one: its id or slug, a
|
|
1167
|
+
# URL among its page fields, or a source path in its frontmatter names an admin-only document;
|
|
1168
|
+
# describes-admin-document: its title, summary, body or other frontmatter links an admin-only
|
|
1169
|
+
# document (path, file name, distinctive slug, hosted-page slug), quotes its distinctive title,
|
|
1170
|
+
# or speaks of an admin-only document ("into one admin-only doc", "(admin-only frontmatter)").
|
|
1171
|
+
# An admin-only document is named by its path, its file stem, the stem without a date prefix, its
|
|
1172
|
+
# id or slug field, and the last part of a URL in its own page fields. Bare names must be distinctive
|
|
1173
|
+
# (``distinctive_slug``) and must not also be the file stem of a document that stays on the map; a
|
|
1174
|
+
# full path or an explicit page field always counts. A title counts when it has three or more words and 20 or more characters
|
|
1175
|
+
# and no shown document carries the same title.
|
|
1176
|
+
PAGE_RULE = "page-of-admin-document"
|
|
1177
|
+
DESCRIBES_RULE = "describes-admin-document"
|
|
1178
|
+
PAGE_FIELD = _lazy(r"^(?:id|slug|url|urls|page|pages|hosted|published|canonical|scroll|artifact|source|sources|"
|
|
1179
|
+
r"derived_from|builds_on|based_on|[a-z]+_url|hosted_[a-z_]+|published_[a-z_]+|source_[a-z_]+)$",
|
|
1180
|
+
re.I)
|
|
1181
|
+
OWN_PAGE_FIELD = _lazy(r"^(?:url|urls|page|hosted|published|[a-z]+_url|hosted_[a-z_]+|published_[a-z_]+)$", re.I)
|
|
1182
|
+
PAGE_LINK = _lazy(r"https?://[^\s\"'<>)\]`|]+")
|
|
1183
|
+
DATE_PREFIX = _lazy(r"^\d{4}-\d{2}-\d{2}-")
|
|
1184
|
+
REFERENCE_TOKEN = _lazy(r"[\w.-]+")
|
|
1185
|
+
PATH_TOKEN = _lazy(r"[\w./-]+")
|
|
1186
|
+
DOC_NOUN = r"(?:docs?|documents?|artifacts?|files?|lists?|memos?|briefings?|overviews?|pages?|versions?|drafts?|decks?|papers?|reports?)"
|
|
1187
|
+
ADMIN_ONLY_NAMED = _lazy(
|
|
1188
|
+
r"\badmin[- ]only\s+" + DOC_NOUN + r"\b|\b" + DOC_NOUN + r"\b[^.\n]{0,40}\(\s*admin[- ]only\b|"
|
|
1189
|
+
r"\b(?:marked|saved|kept|flagged|filed|committed)\b[^.\n]{0,60}\badmin[- ]only\b|"
|
|
1190
|
+
r"\badmin[- ]only\s+frontmatter\b", re.I)
|
|
1191
|
+
TITLE_QUOTE_WORDS, TITLE_QUOTE_CHARS = 3, 20
|
|
1192
|
+
|
|
1193
|
+
|
|
1194
|
+
def reference_name(value: str) -> str:
|
|
1195
|
+
"""A path, file name or URL reduced to its last part, lower case, without .md or .html."""
|
|
1196
|
+
value = value.strip().rstrip(".,;:!?)]'\"`").split("#", 1)[0].split("?", 1)[0]
|
|
1197
|
+
value = unquote(value)
|
|
1198
|
+
return re.sub(r"\.(?:md|html?)$", "", value.rstrip("/").rsplit("/", 1)[-1], flags=re.I).lower()
|
|
1199
|
+
|
|
1200
|
+
|
|
1201
|
+
def title_key(text: str) -> str:
|
|
1202
|
+
return " ".join(re.findall(r"[^\W_]+", fold.__wrapped__(text or ""))) # uncached: whole documents
|
|
1203
|
+
|
|
1204
|
+
|
|
1205
|
+
class AdminReferences:
|
|
1206
|
+
"""Names, paths and titles of the admin-only documents, and the check against one record."""
|
|
1207
|
+
|
|
1208
|
+
def __init__(self, admin_docs, kept_docs):
|
|
1209
|
+
self.paths: set[str] = set()
|
|
1210
|
+
self.page_names: set[str] = set()
|
|
1211
|
+
self.names: set[str] = set()
|
|
1212
|
+
self.titles: set[str] = set()
|
|
1213
|
+
candidate_names, candidate_titles = set(), set()
|
|
1214
|
+
for rel, fm, title in admin_docs:
|
|
1215
|
+
self.paths.update({rel.lower(), rel.lower().removesuffix(".md")})
|
|
1216
|
+
stem = Path(rel).stem.lower()
|
|
1217
|
+
names = {stem, DATE_PREFIX.sub("", stem)}
|
|
1218
|
+
for key in ("id", "slug"):
|
|
1219
|
+
if isinstance(fm.get(key), str):
|
|
1220
|
+
names.add(reference_name(fm[key]))
|
|
1221
|
+
for key, value in fm.items():
|
|
1222
|
+
if OWN_PAGE_FIELD.match(str(key)):
|
|
1223
|
+
names.update(reference_name(link) for text in strings_in(value) for link in PAGE_LINK.findall(text))
|
|
1224
|
+
# Explicit page metadata is a direct source reference even for a short slug or
|
|
1225
|
+
# when its derivative uses the same stem. Only prose references need ambiguity guards.
|
|
1226
|
+
self.page_names.update(n for n in names if n)
|
|
1227
|
+
candidate_names.update(n for n in names if distinctive_slug(n))
|
|
1228
|
+
key = title_key(title)
|
|
1229
|
+
if len(key.split()) >= TITLE_QUOTE_WORDS and len(key) >= TITLE_QUOTE_CHARS:
|
|
1230
|
+
candidate_titles.add(key)
|
|
1231
|
+
# An already identified page of a private source cannot make that source's names
|
|
1232
|
+
# ambiguous. Only independently eligible records may preserve shared vocabulary.
|
|
1233
|
+
independent_docs = [doc for doc in kept_docs if not self.page(doc.get("fm") or {})]
|
|
1234
|
+
kept_stems = {s for doc in independent_docs for s in (Path(doc["rel"]).stem.lower(),
|
|
1235
|
+
DATE_PREFIX.sub("", Path(doc["rel"]).stem.lower()))}
|
|
1236
|
+
kept_titles = {title_key(doc.get("title") or "") for doc in independent_docs}
|
|
1237
|
+
self.names = candidate_names - kept_stems
|
|
1238
|
+
self.titles = candidate_titles - kept_titles
|
|
1239
|
+
self.digest = hashlib.sha256("\n".join(sorted(self.paths) + ["--"] + sorted(self.page_names) + ["--"] + sorted(self.names) + ["--"]
|
|
1240
|
+
+ sorted(self.titles)).encode()).hexdigest()
|
|
1241
|
+
|
|
1242
|
+
def _names_value(self, value: str) -> bool:
|
|
1243
|
+
value = value.strip()
|
|
1244
|
+
path = value.removeprefix("memory/").lower().rstrip("/")
|
|
1245
|
+
return path in self.paths or reference_name(value) in self.page_names
|
|
1246
|
+
|
|
1247
|
+
def page(self, fm: dict) -> bool:
|
|
1248
|
+
for key, value in fm.items():
|
|
1249
|
+
if not PAGE_FIELD.match(str(key)):
|
|
1250
|
+
continue
|
|
1251
|
+
for text in strings_in(value):
|
|
1252
|
+
links = PAGE_LINK.findall(text)
|
|
1253
|
+
if any(reference_name(link) in self.page_names for link in links):
|
|
1254
|
+
return True
|
|
1255
|
+
if not links and len(text) <= 300 and self._names_value(text):
|
|
1256
|
+
return True
|
|
1257
|
+
return False
|
|
1258
|
+
|
|
1259
|
+
def describes(self, title: str, summary: str, body: str, fm: dict) -> bool:
|
|
1260
|
+
text = "\n".join([title or "", summary or "", body or ""] + [s for v in fm.values() for s in strings_in(v)])
|
|
1261
|
+
_work(text)
|
|
1262
|
+
if any(reference_name(link) in self.page_names for link in PAGE_LINK.findall(text)):
|
|
1263
|
+
return True
|
|
1264
|
+
for match in PATH_TOKEN.finditer(text):
|
|
1265
|
+
token = match.group().strip("./").removeprefix("memory/").lower()
|
|
1266
|
+
if "/" in token and (token in self.paths or token.removesuffix(".md") in self.paths):
|
|
1267
|
+
return True
|
|
1268
|
+
for match in REFERENCE_TOKEN.finditer(text):
|
|
1269
|
+
if text[match.end():match.end() + 1] == "/" or text[max(0, match.start() - 1):match.start()] == "~":
|
|
1270
|
+
continue # a folder, or a home-relative name
|
|
1271
|
+
if reference_name(match.group()) in self.names:
|
|
1272
|
+
return True
|
|
1273
|
+
if self.titles:
|
|
1274
|
+
words = " " + title_key(text) + " "
|
|
1275
|
+
for key in self.titles:
|
|
1276
|
+
_work(words)
|
|
1277
|
+
if " " + key + " " in words:
|
|
1278
|
+
return True
|
|
1279
|
+
return bool(ADMIN_ONLY_NAMED.search(text))
|
|
1280
|
+
|
|
1281
|
+
|
|
1282
|
+
def admin_shingles(admin_docs) -> dict[int, object]:
|
|
1283
|
+
"""Word-gram -> the admin-only document holding it (an int), or a tuple when several do."""
|
|
1284
|
+
index: dict[int, object] = {}
|
|
1285
|
+
for k, (_rel, _fm, _title, body, _sha) in enumerate(admin_docs):
|
|
1286
|
+
for gram in shingles(body):
|
|
1287
|
+
held = index.get(gram)
|
|
1288
|
+
if held is None:
|
|
1289
|
+
index[gram] = k
|
|
1290
|
+
elif held != k and (isinstance(held, int) or k not in held):
|
|
1291
|
+
index[gram] = (held, k) if isinstance(held, int) else held + (k,)
|
|
1292
|
+
return index
|
|
1293
|
+
|
|
1294
|
+
|
|
1295
|
+
def copy_of_admin(body: str, index: dict[int, object]) -> bool:
|
|
1296
|
+
grams = shingles(body)
|
|
1297
|
+
if len(grams) < COPY_MIN_SHINGLES or not index:
|
|
1298
|
+
return False
|
|
1299
|
+
inside: Counter = Counter()
|
|
1300
|
+
for gram in grams:
|
|
1301
|
+
held = index.get(gram)
|
|
1302
|
+
if held is None:
|
|
1303
|
+
continue
|
|
1304
|
+
inside.update((held,) if isinstance(held, int) else held)
|
|
1305
|
+
return bool(inside) and max(inside.values()) >= COPY_SHARE * len(grams)
|
|
1306
|
+
|
|
1307
|
+
|
|
1308
|
+
def derivative_rule(doc: dict, references: AdminReferences, index: dict[int, object]) -> str:
|
|
1309
|
+
"""copy-, page- or describes-admin-document for one parsed document, or ""."""
|
|
1310
|
+
_work(doc["body"])
|
|
1311
|
+
fm = doc.get("fm") or {}
|
|
1312
|
+
if copy_of_admin(doc["body"], index):
|
|
1313
|
+
return COPY_RULE
|
|
1314
|
+
if references.page(fm):
|
|
1315
|
+
return PAGE_RULE
|
|
1316
|
+
summary = doc.get("summary")
|
|
1317
|
+
if summary is None:
|
|
1318
|
+
summary = strip_memory_paths(summary_for(fm, doc["body"]))
|
|
1319
|
+
return DESCRIBES_RULE if references.describes(doc.get("title"), summary, doc["body"], fm) else ""
|
|
1320
|
+
|
|
1321
|
+
|
|
1322
|
+
def derivatives(admin_docs, docs, known=None, released=frozenset(), learn=None, shields=None) -> dict[str, tuple[str | None, str]]:
|
|
1323
|
+
"""rel -> (copy-, page- or describes-admin-document or None, key) for ``docs`` (dicts of rel, fm,
|
|
1324
|
+
title, summary, body) against ``admin_docs`` ((rel, fm, title, body, sha256)). Names are recomputed
|
|
1325
|
+
after each removal, so a derivative held in one pass shields no name in the next; an unchanged key
|
|
1326
|
+
(the admin set and its names) ends the run. ``known(rel, key)`` returns a kept verdict or None,
|
|
1327
|
+
``learn`` hears every one; ``released`` documents stay shown; only ``shields`` make a name ambiguous."""
|
|
1328
|
+
admin_refs = [(rel, fm, title) for rel, fm, title, _body, _sha in admin_docs]
|
|
1329
|
+
admin_hashes = [f"{rel}\t{sha}" for rel, _fm, _title, _body, sha in admin_docs]
|
|
1330
|
+
found: dict[str, tuple[str | None, str]] = {}
|
|
1331
|
+
index = None
|
|
1332
|
+
candidates, key = list(docs), None
|
|
1333
|
+
while True:
|
|
1334
|
+
if (budget := _WORK_BUDGET.get()) is not None:
|
|
1335
|
+
budget.charge("passes")
|
|
1336
|
+
for rel, _fm, title in admin_refs:
|
|
1337
|
+
_work(rel)
|
|
1338
|
+
_work(title or "")
|
|
1339
|
+
for doc in candidates:
|
|
1340
|
+
_work(doc["rel"])
|
|
1341
|
+
_work(doc.get("title") or "")
|
|
1342
|
+
references = AdminReferences(admin_refs, [doc for doc in candidates if not shields or shields(doc)])
|
|
1343
|
+
last, key = key, hashlib.sha256("\n".join(admin_hashes + [references.digest]).encode()).hexdigest()
|
|
1344
|
+
if key == last:
|
|
1345
|
+
return found
|
|
1346
|
+
kept = []
|
|
1347
|
+
for doc in candidates:
|
|
1348
|
+
rule = known(doc["rel"], key) if known else None
|
|
1349
|
+
fresh = rule is None
|
|
1350
|
+
if fresh:
|
|
1351
|
+
if index is None:
|
|
1352
|
+
index = admin_shingles(admin_docs)
|
|
1353
|
+
rule = derivative_rule(doc, references, index) or (
|
|
1354
|
+
derivative_rule(doc["spelled"], references, index) if doc.get("spelled") else "")
|
|
1355
|
+
if learn:
|
|
1356
|
+
learn(doc["rel"], key, rule, fresh)
|
|
1357
|
+
found[doc["rel"]] = (rule or None, key)
|
|
1358
|
+
if not rule or doc["rel"] in released:
|
|
1359
|
+
kept.append(doc)
|
|
1360
|
+
if len(kept) == len(candidates):
|
|
1361
|
+
return found
|
|
1362
|
+
candidates = kept
|
|
1363
|
+
|
|
1364
|
+
|
|
1365
|
+
# The exporter's curation (inventory, captures, noise), copied: a document off the members map never
|
|
1366
|
+
# makes an admin name ambiguous there, so it does not here. test_held.py fails when they drift apart.
|
|
1367
|
+
SKIP_DIRS = {"notes", "scratch", "identity", "onboarding", "diagnostics", "analytics",
|
|
1368
|
+
"todos", "lifecycle", "comments", "conversations", "dreams"}
|
|
1369
|
+
SKIP_NAMES = {"index.md", "readme.md", "memory.md"}
|
|
1370
|
+
PEOPLE_DIRS = {"people", "participants"}
|
|
1371
|
+
GENERATED_FILE = _lazy(r"(?:^|/)artifacts/(?:[^/]+/)*[^/]*-session-dashboard\.md$"
|
|
1372
|
+
r"|(?:^|/)sessions/(?:[^/]+/)*\d{2}-[^/-]+-session\.md$", re.I)
|
|
1373
|
+
GENERATED_KINDS = {"ripcord"}
|
|
1374
|
+
NOISE_SUBJECT = _lazy(r"^(?:wrap: )?ripcord\b|\b(?:raw (?:meeting|call) transcript|test handoff|delete me|page withdrawn)\b", re.I)
|
|
1375
|
+
|
|
1376
|
+
|
|
1377
|
+
def noise_rule(title, fm, rel):
|
|
1378
|
+
# Imported corpora are raw source material, separate from the organization's work.
|
|
1379
|
+
parts = Path(rel.removeprefix("memory/")).parts
|
|
1380
|
+
if parts and parts[0] == "corpora":
|
|
1381
|
+
return "raw-corpus"
|
|
1382
|
+
if NOISE_SUBJECT.search(title) or {str(fm.get(key) or "").lower() for key in ("type", "kind")} & {
|
|
1383
|
+
"raw-transcript", "raw_transcript", "interruption", "session-interruption"}:
|
|
1384
|
+
return "raw-capture-or-placeholder"
|
|
1385
|
+
return None
|
|
1386
|
+
|
|
1387
|
+
|
|
1388
|
+
def on_map(doc: dict) -> bool:
|
|
1389
|
+
"""Whether the members map shows a document its rules leave shown (export.py inventory and parse)."""
|
|
1390
|
+
from .eligibility import is_internal_event
|
|
1391
|
+
|
|
1392
|
+
rel, fm = doc["rel"], doc.get("fm") or {}
|
|
1393
|
+
parts, name = rel.split("/"), rel.rsplit("/", 1)[-1].lower()
|
|
1394
|
+
kinds = {str(fm.get(key) or "").strip().lower() for key in ("kind", "type")}
|
|
1395
|
+
return not (any(part.startswith(".") for part in parts) or any(part in PEOPLE_DIRS for part in parts[:-1])
|
|
1396
|
+
or (len(parts) > 1 and parts[0] in SKIP_DIRS) or name in SKIP_NAMES or name.startswith("waitlist")
|
|
1397
|
+
or GENERATED_FILE.search(rel) or kinds & GENERATED_KINDS or noise_rule(doc["title"], fm, rel)
|
|
1398
|
+
or is_internal_event("memory/" + rel, fm.get("type") if isinstance(fm.get("type"), str) else None))
|
|
1399
|
+
|
|
1400
|
+
|
|
1401
|
+
# Each rule's names, so a verdict read back from the cache keeps its class.
|
|
1402
|
+
RULE_CLASS = {**{name: 2 for name, _pattern in PAPERWORK_RULES}, "personnel-or-equity": 3, "raw-transcript": 4,
|
|
1403
|
+
"valuation-or-dilution": 5, "fundraising-figures": 5, "restricted-audience": 6,
|
|
1404
|
+
"restricted-marking": 6, "not-for-sharing": 6, "personal-capture": 7, "access-credential": 7,
|
|
1405
|
+
"contract-lifecycle": 7, "private-purpose": 7, "personnel-change": 8, "investor-outcome": 8,
|
|
1406
|
+
"interpersonal-conflict": 8, COPY_RULE: 9, PAGE_RULE: 9, DESCRIBES_RULE: 9}
|
|
1407
|
+
|
|
1408
|
+
|
|
1409
|
+
# -- Verdicts over a memory tree, and their cache -----------------------------------------------
|
|
1410
|
+
class Verdicts:
|
|
1411
|
+
"""``held``: memory-relative path -> "admin" or the rule that holds it; ``computed``: the paths
|
|
1412
|
+
classified in this call rather than read from the cache; ``epoch``: holds_epoch over ``held``."""
|
|
1413
|
+
|
|
1414
|
+
__slots__ = ("held", "computed", "documents", "epoch")
|
|
1415
|
+
|
|
1416
|
+
def __init__(self, held: Mapping[str, str], computed: frozenset[str], documents: int, epoch: str):
|
|
1417
|
+
self.held, self.computed, self.documents, self.epoch = held, computed, documents, epoch
|
|
1418
|
+
|
|
1419
|
+
|
|
1420
|
+
def cache_path(memory_root: Path) -> Path:
|
|
1421
|
+
"""This user's private hint for one memory, beside its shadow counts and lifecycle state."""
|
|
1422
|
+
key = hashlib.sha256(str(Path(memory_root).resolve()).encode("utf-8")).hexdigest()[:16]
|
|
1423
|
+
return Path.home() / ".egregore" / "runtime" / "held" / f"{key}.json"
|
|
1424
|
+
|
|
1425
|
+
|
|
1426
|
+
def _documents(root: Path):
|
|
1427
|
+
"""(path as named, path it resolves to, its stat) for every Markdown document under ``root``, as ``os.walk``
|
|
1428
|
+
lists them: dot names skipped, a folder link never followed, a file link only to a regular file inside ``root``.
|
|
1429
|
+
One ``lstat`` per file and no ``Path`` per entry: this walk is every Runtime process's freshness check."""
|
|
1430
|
+
base = str(root)
|
|
1431
|
+
folders = [""]
|
|
1432
|
+
while folders:
|
|
1433
|
+
folder = folders.pop()
|
|
1434
|
+
try:
|
|
1435
|
+
with os.scandir(os.path.join(base, folder) if folder else base) as listing:
|
|
1436
|
+
found = sorted(listing, key=lambda entry: entry.name)
|
|
1437
|
+
except OSError:
|
|
1438
|
+
continue
|
|
1439
|
+
for entry in found:
|
|
1440
|
+
name = entry.name
|
|
1441
|
+
if name.startswith("."):
|
|
1442
|
+
continue
|
|
1443
|
+
rel = f"{folder}/{name}" if folder else name
|
|
1444
|
+
try:
|
|
1445
|
+
is_folder = entry.is_dir()
|
|
1446
|
+
except OSError:
|
|
1447
|
+
is_folder = False
|
|
1448
|
+
if is_folder:
|
|
1449
|
+
if not entry.is_symlink():
|
|
1450
|
+
folders.append(rel)
|
|
1451
|
+
continue
|
|
1452
|
+
if not name.lower().endswith(".md"):
|
|
1453
|
+
continue
|
|
1454
|
+
try:
|
|
1455
|
+
info, target = entry.stat(follow_symlinks=False), rel
|
|
1456
|
+
if stat.S_ISLNK(info.st_mode):
|
|
1457
|
+
resolved = Path(entry.path).resolve(strict=True)
|
|
1458
|
+
if not resolved.is_relative_to(root):
|
|
1459
|
+
continue
|
|
1460
|
+
info, target = resolved.stat(), resolved.relative_to(root).as_posix()
|
|
1461
|
+
except (OSError, RuntimeError, ValueError):
|
|
1462
|
+
continue
|
|
1463
|
+
if stat.S_ISREG(info.st_mode):
|
|
1464
|
+
yield rel, target, info
|
|
1465
|
+
|
|
1466
|
+
|
|
1467
|
+
def _read_cache(path: Path) -> dict:
|
|
1468
|
+
try:
|
|
1469
|
+
data = json.loads(Path(path).read_text(encoding="utf-8"))
|
|
1470
|
+
entries = data.get("entries") if data.get("version") == 1 else None
|
|
1471
|
+
except (OSError, ValueError, AttributeError):
|
|
1472
|
+
return {}
|
|
1473
|
+
return entries if isinstance(entries, dict) else {}
|
|
1474
|
+
|
|
1475
|
+
|
|
1476
|
+
def _write_private(path: Path, payload: dict) -> None:
|
|
1477
|
+
path = Path(path)
|
|
1478
|
+
temporary = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
|
1479
|
+
try:
|
|
1480
|
+
path.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
1481
|
+
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_TRUNC | os.O_NOFOLLOW, 0o600)
|
|
1482
|
+
with os.fdopen(descriptor, "w", encoding="utf-8") as handle:
|
|
1483
|
+
json.dump(payload, handle, separators=(",", ":"))
|
|
1484
|
+
temporary.replace(path)
|
|
1485
|
+
except OSError:
|
|
1486
|
+
pass # A hint only: no verdict depends on it being written.
|
|
1487
|
+
|
|
1488
|
+
|
|
1489
|
+
def _write_cache(path: Path, entries: dict) -> None:
|
|
1490
|
+
_write_private(path, {"version": 1, "entries": entries})
|
|
1491
|
+
|
|
1492
|
+
|
|
1493
|
+
def _verdict(value) -> str | None:
|
|
1494
|
+
"""A cached verdict, or None when the hint is not one this module could have written."""
|
|
1495
|
+
return value if isinstance(value, str) and (not value or value in RULE_CLASS) else None
|
|
1496
|
+
|
|
1497
|
+
|
|
1498
|
+
def _prefixes(release: Iterable[str]) -> tuple[str, ...]:
|
|
1499
|
+
"""``access.release`` paths and folders, relative to memory ("" is all of it)."""
|
|
1500
|
+
return tuple(sorted({item.strip().strip("/").removeprefix("memory").strip("/") for item in release if isinstance(item, str)}))
|
|
1501
|
+
|
|
1502
|
+
|
|
1503
|
+
class Tree:
|
|
1504
|
+
"""One memory tree, as walked or as committed: class 1 and the rules per document, then the admin
|
|
1505
|
+
derivatives among the documents still shown, to the exporter's fixpoint. ``entries``: rel ->
|
|
1506
|
+
(target, sha256, verdict) before releases; ``held``: after them (a rule verdict is lifted when both
|
|
1507
|
+
spellings are released, class 1 never); ``computed``: classified now, not taken from ``cache``."""
|
|
1508
|
+
|
|
1509
|
+
def __init__(self, documents: Iterable[tuple[str, str, bytes]], *, release: Iterable[str] = (),
|
|
1510
|
+
cache: Path | None = None):
|
|
1511
|
+
prefixes = _prefixes(release)
|
|
1512
|
+
|
|
1513
|
+
def lifted(rel: str) -> bool:
|
|
1514
|
+
return any(not p or rel == p or rel.startswith(p + "/") for p in prefixes)
|
|
1515
|
+
|
|
1516
|
+
docs = sorted((rel, target, hashlib.sha256(raw).hexdigest(), raw.decode("utf-8", errors="replace"))
|
|
1517
|
+
for rel, target, raw in documents)
|
|
1518
|
+
admin_set = hashlib.sha256("\n".join(f"{rel}\t{sha}" for rel, _, sha, text in docs
|
|
1519
|
+
if document_is_admin(text)).encode()).hexdigest()
|
|
1520
|
+
release_digest = hashlib.sha256(json.dumps(list(prefixes)).encode()).hexdigest()
|
|
1521
|
+
stored = _read_cache(cache) if cache is not None else {}
|
|
1522
|
+
entries: dict[str, str] = {}
|
|
1523
|
+
computed: set[str] = set()
|
|
1524
|
+
self.entries: dict[str, tuple[str, str, str]] = {}
|
|
1525
|
+
self.held: dict[str, str] = {}
|
|
1526
|
+
self.documents = len(docs)
|
|
1527
|
+
self.listed = tuple((rel, target) for rel, target, *_ in docs) # (path as named, path it reads as)
|
|
1528
|
+
self._admin_docs: list = []
|
|
1529
|
+
self._shown: list[dict] = []
|
|
1530
|
+
free: set[str] = set()
|
|
1531
|
+
for rel, target, sha, text in docs:
|
|
1532
|
+
_work(text)
|
|
1533
|
+
if document_is_admin(text):
|
|
1534
|
+
doc = _parse(rel, text, summary=False)
|
|
1535
|
+
self._admin_docs.append((rel, doc["fm"], doc["title"], doc["body"], sha))
|
|
1536
|
+
self.entries[rel] = (target, sha, "admin")
|
|
1537
|
+
self.held[rel] = "admin"
|
|
1538
|
+
continue
|
|
1539
|
+
key = hashlib.sha256(json.dumps([rel, target, sha, HELD_RULES_VERSION, admin_set,
|
|
1540
|
+
release_digest]).encode()).hexdigest()
|
|
1541
|
+
verdict = _verdict(stored.get(key))
|
|
1542
|
+
doc = None
|
|
1543
|
+
if verdict is None:
|
|
1544
|
+
doc = _parse(rel, text)
|
|
1545
|
+
verdict = (document_rule(rel, text, doc["fm"], doc["body"], doc["title"], doc["summary"])
|
|
1546
|
+
or (target != rel and _rule(target, text)) or "")
|
|
1547
|
+
computed.add(rel)
|
|
1548
|
+
entries[key] = verdict
|
|
1549
|
+
self.entries[rel] = (target, sha, verdict)
|
|
1550
|
+
if lifted(rel) and lifted(target):
|
|
1551
|
+
free.add(rel)
|
|
1552
|
+
elif verdict:
|
|
1553
|
+
self.held[rel] = verdict
|
|
1554
|
+
continue
|
|
1555
|
+
doc = doc or _parse(rel, text, summary=False)
|
|
1556
|
+
spelled = None if target == rel else {**doc, "rel": target, "title": title_for(doc["fm"], doc["body"], target)}
|
|
1557
|
+
self._shown.append({**doc, "target": target, "sha": sha, "spelled": spelled})
|
|
1558
|
+
|
|
1559
|
+
def name(doc: dict, key: str) -> str:
|
|
1560
|
+
return hashlib.sha256(json.dumps(["derived", doc["rel"], doc["target"], doc["sha"], HELD_RULES_VERSION,
|
|
1561
|
+
key]).encode()).hexdigest()
|
|
1562
|
+
|
|
1563
|
+
shown = {doc["rel"]: doc for doc in self._shown}
|
|
1564
|
+
|
|
1565
|
+
def learn(rel: str, key: str, rule: str, fresh: bool) -> None:
|
|
1566
|
+
entries[name(shown[rel], key)] = rule
|
|
1567
|
+
if fresh:
|
|
1568
|
+
computed.add(rel)
|
|
1569
|
+
|
|
1570
|
+
found = derivatives(self._admin_docs, self._shown, lambda rel, key: _verdict(stored.get(name(shown[rel], key))),
|
|
1571
|
+
free, learn, on_map)
|
|
1572
|
+
for rel, (rule, _key) in found.items():
|
|
1573
|
+
if rule and not self.entries[rel][2]:
|
|
1574
|
+
self.entries[rel] = (*self.entries[rel][:2], rule)
|
|
1575
|
+
if rel not in free:
|
|
1576
|
+
self.held[rel] = rule
|
|
1577
|
+
self._judge, self.renew = None, None
|
|
1578
|
+
# What a later document is judged against: the names of the documents still shown, not their text.
|
|
1579
|
+
self._shown = [{"rel": doc["rel"], "fm": doc["fm"], "title": doc["title"]}
|
|
1580
|
+
for doc in self._shown if doc["rel"] not in self.held]
|
|
1581
|
+
self.computed = frozenset(computed)
|
|
1582
|
+
if cache is not None and (computed or len(entries) != len(stored)):
|
|
1583
|
+
_write_cache(cache, entries)
|
|
1584
|
+
|
|
1585
|
+
def _derived(self, doc: dict, read_as: tuple[str, ...] = ()) -> str:
|
|
1586
|
+
"""Whether ``doc`` derives from this tree's admin-only set. ``read_as`` names the spellings whose bytes
|
|
1587
|
+
``doc`` is when they differ from this tree's: their copies here shield no name, and the document as read
|
|
1588
|
+
shields as the exporter would read it, so a newer copy cannot hide the bytes a caller read."""
|
|
1589
|
+
admins = [(r, fm, title) for r, fm, title, _b, _s in self._admin_docs]
|
|
1590
|
+
if self._judge is None:
|
|
1591
|
+
self._judge = (AdminReferences(admins, [shown for shown in self._shown if on_map(shown)]),
|
|
1592
|
+
admin_shingles(self._admin_docs))
|
|
1593
|
+
references, index = self._judge
|
|
1594
|
+
if read_as:
|
|
1595
|
+
kept = [shown for shown in self._shown if shown["rel"] not in read_as] + [
|
|
1596
|
+
{"rel": name, "fm": doc["fm"], "title": title_for(doc["fm"], doc["body"], name)} for name in read_as]
|
|
1597
|
+
references = AdminReferences(admins, [shown for shown in kept if on_map(shown)])
|
|
1598
|
+
return derivative_rule(doc, references, index)
|
|
1599
|
+
|
|
1600
|
+
def _entry(self, rel: str) -> tuple[str, str, str] | None:
|
|
1601
|
+
return self.entries.get(rel)
|
|
1602
|
+
|
|
1603
|
+
def _spelled(self, entry: tuple[str, str, str], target: str | None) -> bool:
|
|
1604
|
+
"""Whether ``target`` is the file this entry resolved to (None: as named)."""
|
|
1605
|
+
return target in (None, entry[0])
|
|
1606
|
+
|
|
1607
|
+
walk = "" # the digest of the walk this tree answers for, when it is one
|
|
1608
|
+
_judged = None
|
|
1609
|
+
|
|
1610
|
+
def verdict(self, rel: str, sha: str, target: str | None = None) -> str | None:
|
|
1611
|
+
"""The verdict this tree holds for these bytes (by sha256) at ``rel``, "" when shown; None when it holds none
|
|
1612
|
+
for them: changed, never walked, or read through another spelling. Class 1 is the caller's to read."""
|
|
1613
|
+
entry = self._entry(rel)
|
|
1614
|
+
return entry[2] if entry is not None and entry[1] == sha and self._spelled(entry, target) else None
|
|
1615
|
+
|
|
1616
|
+
def _remembered(self) -> "_Judged":
|
|
1617
|
+
if self._judged is None:
|
|
1618
|
+
self._judged = _Judged(None, self.walk)
|
|
1619
|
+
return self._judged
|
|
1620
|
+
|
|
1621
|
+
def known(self, rel: str, sha: str, target: str | None = None) -> tuple[int, ...] | None:
|
|
1622
|
+
"""The classes this tree already judged for these bytes at ``rel`` (not class 1), without renewing or
|
|
1623
|
+
deriving: its verdict, or a judgement kept for bytes it holds none for. None when neither exists."""
|
|
1624
|
+
verdict = self.verdict(rel, sha, target)
|
|
1625
|
+
if verdict is not None:
|
|
1626
|
+
return (RULE_CLASS[verdict],) if verdict in RULE_CLASS else ()
|
|
1627
|
+
return self._remembered().get(rel, target, sha)
|
|
1628
|
+
|
|
1629
|
+
def classes(self, rel: str, raw: bytes | None, target: str | None = None, renew: bool = True) -> tuple[int, ...]:
|
|
1630
|
+
"""The class of one document as read now: its verdict here while its bytes are unchanged. A document
|
|
1631
|
+
walked since, or changed, renews a walked tree first; otherwise its rules under both spellings and
|
|
1632
|
+
a derivative check against this tree's admin-only set decide."""
|
|
1633
|
+
if raw is None:
|
|
1634
|
+
return ()
|
|
1635
|
+
text = raw.decode("utf-8", errors="replace")
|
|
1636
|
+
if document_is_admin(text): # class 1 is read from the bytes on every call
|
|
1637
|
+
return (1,)
|
|
1638
|
+
entry, sha = self._entry(rel), hashlib.sha256(raw).hexdigest()
|
|
1639
|
+
if entry is not None and entry[1] == sha and self._spelled(entry, target):
|
|
1640
|
+
return (RULE_CLASS[entry[2]],) if entry[2] in RULE_CLASS else ()
|
|
1641
|
+
linked = self._entry(target) if entry is None and target not in (None, rel) else None
|
|
1642
|
+
if linked is not None and linked[1] == sha:
|
|
1643
|
+
verdict = linked[2] if linked[2] in RULE_CLASS else ""
|
|
1644
|
+
return text_classes(text, rel) or ((RULE_CLASS[verdict],) if verdict else ()) or (
|
|
1645
|
+
(9,) if self._derived(_parse(rel, text)) else ())
|
|
1646
|
+
walked = rel.lower().endswith(".md") and not any(part.startswith(".") for part in rel.split("/"))
|
|
1647
|
+
if renew and self.renew and (entry is not None or walked):
|
|
1648
|
+
return self.renew().classes(rel, raw, target, renew=False)
|
|
1649
|
+
kept = self._remembered().get(rel, target, sha) # judged here before: a function of this walk and these bytes
|
|
1650
|
+
if kept is not None:
|
|
1651
|
+
return kept
|
|
1652
|
+
found = text_classes(text, rel) or (text_classes(text, target) if target not in (None, rel) else ())
|
|
1653
|
+
spellings = (rel,) if target in (None, rel) else (rel, target)
|
|
1654
|
+
read_as = spellings if walked else () # these bytes stand in for this tree's copy of the document
|
|
1655
|
+
found = found or ((9,) if any(self._derived(_parse(name, text), read_as) for name in spellings) else ())
|
|
1656
|
+
self._remembered().put(rel, target, sha, found)
|
|
1657
|
+
return found
|
|
1658
|
+
|
|
1659
|
+
|
|
1660
|
+
_TREES: dict[tuple, tuple[float, str, Tree | None]] = {}
|
|
1661
|
+
FRESH_SECONDS = 2.0 # a walked tree answers this long before memory is walked again
|
|
1662
|
+
INDEX_VERSION = 1
|
|
1663
|
+
# A file changed this shortly before a walk can change again within its file system's timestamp granularity (a
|
|
1664
|
+
# second or two on some, a coarse clock tick on others) and keep its stat. Before an index that saw such a file is
|
|
1665
|
+
# trusted the file is hashed again, as git treats a racily clean index entry.
|
|
1666
|
+
RACY_NS = 2_000_000_000
|
|
1667
|
+
TURN_SECONDS = 120.0 # how long a build waits for another process building the same memory, then builds too
|
|
1668
|
+
_now = time.time_ns
|
|
1669
|
+
|
|
1670
|
+
|
|
1671
|
+
_ROOTS: dict[str, tuple[float, Path]] = {}
|
|
1672
|
+
|
|
1673
|
+
|
|
1674
|
+
def _resolved(memory_root: Path) -> Path:
|
|
1675
|
+
"""The memory root resolved, again after FRESH_SECONDS: a policy check asks for it on every read."""
|
|
1676
|
+
name, now = str(memory_root), time.monotonic()
|
|
1677
|
+
seen = _ROOTS.get(name)
|
|
1678
|
+
if seen is None or now - seen[0] >= FRESH_SECONDS:
|
|
1679
|
+
seen = _ROOTS[name] = (now, Path(memory_root).resolve())
|
|
1680
|
+
return seen[1]
|
|
1681
|
+
|
|
1682
|
+
|
|
1683
|
+
def _remember(key: tuple, digest: str, build, walked: float) -> Tree | None:
|
|
1684
|
+
seen = _TREES.get(key)
|
|
1685
|
+
if seen is None or seen[1] != digest:
|
|
1686
|
+
built = build()
|
|
1687
|
+
if built is None:
|
|
1688
|
+
return None
|
|
1689
|
+
while len(_TREES) >= 4:
|
|
1690
|
+
_TREES.pop(next(iter(_TREES)))
|
|
1691
|
+
seen = (walked, digest, built)
|
|
1692
|
+
_TREES[key] = (walked, digest, seen[2])
|
|
1693
|
+
return seen[2]
|
|
1694
|
+
|
|
1695
|
+
|
|
1696
|
+
def _hashed(rel: str) -> str:
|
|
1697
|
+
"""A path as the verdict index names it: like the hint, the index names no path."""
|
|
1698
|
+
return hashlib.sha256(b"held-index\0" + rel.encode("utf-8", "surrogateescape")).hexdigest()[:32]
|
|
1699
|
+
|
|
1700
|
+
|
|
1701
|
+
def index_path(memory_root: Path) -> Path:
|
|
1702
|
+
"""This user's verdict index for one memory, beside its hint: hashed paths, content digests and verdicts."""
|
|
1703
|
+
hint = cache_path(memory_root)
|
|
1704
|
+
return hint.with_name(hint.stem + ".index.json")
|
|
1705
|
+
|
|
1706
|
+
|
|
1707
|
+
def _judged_path(memory_root: Path) -> Path:
|
|
1708
|
+
hint = cache_path(memory_root)
|
|
1709
|
+
return hint.with_name(hint.stem + ".judged.json")
|
|
1710
|
+
|
|
1711
|
+
|
|
1712
|
+
class _Judged:
|
|
1713
|
+
"""Classes a tree judged for bytes its walk holds no verdict for (a document in a dot folder, an unwalked
|
|
1714
|
+
spelling, bytes changed since the walk), by hashed spellings and content digest. They hold for that walk
|
|
1715
|
+
alone; with a ``path`` (this user's index folder) they are kept for later processes on the same walk."""
|
|
1716
|
+
|
|
1717
|
+
def __init__(self, path: Path | None, walk: str):
|
|
1718
|
+
self.path, self.walk, self.entries, self.added = path, walk, {}, False
|
|
1719
|
+
self.loaded = path is None or not walk
|
|
1720
|
+
|
|
1721
|
+
@staticmethod
|
|
1722
|
+
def _key(rel: str, target: str | None, sha: str) -> str:
|
|
1723
|
+
return f"{_hashed(rel)}:{_hashed(target or rel)}:{sha}"
|
|
1724
|
+
|
|
1725
|
+
def _stored(self) -> dict:
|
|
1726
|
+
try:
|
|
1727
|
+
data = json.loads(self.path.read_text(encoding="utf-8"))
|
|
1728
|
+
entries = data.get("entries") if data.get("version") == 1 and data.get("walk") == self.walk else None
|
|
1729
|
+
except (OSError, ValueError, AttributeError):
|
|
1730
|
+
return {}
|
|
1731
|
+
return {key: tuple(value) for key, value in (entries.items() if isinstance(entries, dict) else ())
|
|
1732
|
+
if isinstance(value, list) and all(type(number) is int and 2 <= number <= 9 for number in value)}
|
|
1733
|
+
|
|
1734
|
+
def get(self, rel: str, target: str | None, sha: str) -> tuple[int, ...] | None:
|
|
1735
|
+
if not self.loaded:
|
|
1736
|
+
self.loaded = True
|
|
1737
|
+
self.entries = {**self._stored(), **self.entries}
|
|
1738
|
+
return self.entries.get(self._key(rel, target, sha))
|
|
1739
|
+
|
|
1740
|
+
def put(self, rel: str, target: str | None, sha: str, found: tuple[int, ...]) -> None:
|
|
1741
|
+
self.entries[self._key(rel, target, sha)] = tuple(found)
|
|
1742
|
+
if self.path is not None and self.walk and not self.added:
|
|
1743
|
+
self.added = True
|
|
1744
|
+
atexit.register(self.save)
|
|
1745
|
+
|
|
1746
|
+
def save(self) -> None:
|
|
1747
|
+
_write_private(self.path, {"version": 1, "walk": self.walk,
|
|
1748
|
+
"entries": {key: list(value) for key, value in {**self._stored(), **self.entries}.items()}})
|
|
1749
|
+
|
|
1750
|
+
|
|
1751
|
+
class _Index(Tree):
|
|
1752
|
+
"""A walk this user already built, read back from the verdict index: each document's verdict by hashed path
|
|
1753
|
+
and content sha256, nothing parsed. Bytes it never judged derive against the full tree of the same walk,
|
|
1754
|
+
built on first need (``full``)."""
|
|
1755
|
+
|
|
1756
|
+
def __init__(self, entries: dict, documents: int, full, pairs: tuple = (), release: tuple[str, ...] = ()):
|
|
1757
|
+
self.entries, self.documents, self.computed = entries, documents, frozenset()
|
|
1758
|
+
self._build, self._full, self._judge, self.renew = full, None, None, None
|
|
1759
|
+
self._release, self._held, self.listed = release, None, pairs
|
|
1760
|
+
|
|
1761
|
+
def _entry(self, rel: str):
|
|
1762
|
+
return self.entries.get(_hashed(rel))
|
|
1763
|
+
|
|
1764
|
+
def _spelled(self, entry, target) -> bool:
|
|
1765
|
+
return target is None or entry[0] == _hashed(target)
|
|
1766
|
+
|
|
1767
|
+
def full(self) -> Tree:
|
|
1768
|
+
if self._full is None:
|
|
1769
|
+
self._full = self._build()
|
|
1770
|
+
return self._full
|
|
1771
|
+
|
|
1772
|
+
@property
|
|
1773
|
+
def held(self) -> dict[str, str]:
|
|
1774
|
+
"""``Tree.held`` from the index and the walk's names, nothing parsed: class 1 always, a rule or
|
|
1775
|
+
derivative verdict unless both spellings are released."""
|
|
1776
|
+
if self._held is None:
|
|
1777
|
+
def lifted(rel: str) -> bool:
|
|
1778
|
+
return any(not p or rel == p or rel.startswith(p + "/") for p in self._release)
|
|
1779
|
+
self._held = {rel: entry[2] for rel, target in self.listed if (entry := self._entry(rel)) and entry[2]
|
|
1780
|
+
and (entry[2] == "admin" or not (lifted(rel) and lifted(target)))}
|
|
1781
|
+
return self._held
|
|
1782
|
+
|
|
1783
|
+
def _derived(self, doc: dict, read_as: tuple[str, ...] = ()) -> str:
|
|
1784
|
+
return self.full()._derived(doc, read_as)
|
|
1785
|
+
|
|
1786
|
+
|
|
1787
|
+
def _write_index(root: Path, digest: str, listed: int, built: Tree) -> None:
|
|
1788
|
+
entries = {_hashed(rel): [_hashed(target), sha, verdict] for rel, (target, sha, verdict) in built.entries.items()}
|
|
1789
|
+
_write_private(index_path(root), {"version": INDEX_VERSION, "digest": digest, "listed_ns": listed,
|
|
1790
|
+
"documents": built.documents, "entries": entries})
|
|
1791
|
+
|
|
1792
|
+
|
|
1793
|
+
def _read_index(root: Path, digest: str, listing: list, listed: int, full, release: tuple[str, ...] = ()) -> _Index | None:
|
|
1794
|
+
"""The index of exactly this walk, or None. A file changed within RACY_NS before the walk the index saw is
|
|
1795
|
+
hashed again first; once every such file has aged past that window the index is re-dated, so later walks
|
|
1796
|
+
hash nothing."""
|
|
1797
|
+
try:
|
|
1798
|
+
data = json.loads(index_path(root).read_text(encoding="utf-8"))
|
|
1799
|
+
if data.get("version") != INDEX_VERSION or data.get("digest") != digest:
|
|
1800
|
+
return None
|
|
1801
|
+
stored, documents, seen = data["entries"], data["documents"], data["listed_ns"]
|
|
1802
|
+
if not isinstance(stored, dict) or type(documents) is not int or type(seen) is not int:
|
|
1803
|
+
return None
|
|
1804
|
+
if documents != len(listing) or len(stored) != len(listing):
|
|
1805
|
+
return None # a build that could not read every document it listed answers for no walk
|
|
1806
|
+
entries = {}
|
|
1807
|
+
for name, value in stored.items():
|
|
1808
|
+
if not (isinstance(value, list) and len(value) == 3 and all(isinstance(part, str) for part in value)
|
|
1809
|
+
and (value[2] == "admin" or _verdict(value[2]) is not None)):
|
|
1810
|
+
return None
|
|
1811
|
+
entries[name] = tuple(value)
|
|
1812
|
+
racy = [(rel, target, ctime) for rel, target, _size, _mtime, ctime, _inode in listing if ctime >= seen - RACY_NS]
|
|
1813
|
+
for rel, target, _ctime in racy:
|
|
1814
|
+
entry = entries.get(_hashed(rel))
|
|
1815
|
+
if entry is None or entry[1] != hashlib.sha256((root / target).read_bytes()).hexdigest():
|
|
1816
|
+
return None
|
|
1817
|
+
except (OSError, ValueError, AttributeError, KeyError, TypeError):
|
|
1818
|
+
return None
|
|
1819
|
+
if racy and all(ctime < listed - RACY_NS for *_, ctime in racy):
|
|
1820
|
+
_write_private(index_path(root), {**data, "listed_ns": listed})
|
|
1821
|
+
return _Index(entries, documents, full, tuple((rel, target) for rel, target, *_ in listing), release)
|
|
1822
|
+
|
|
1823
|
+
|
|
1824
|
+
@contextmanager
|
|
1825
|
+
def _turn(root: Path, wait: float = TURN_SECONDS):
|
|
1826
|
+
"""This memory's build turn: a private lock beside its index, waited for up to ``wait`` seconds. Past that, or
|
|
1827
|
+
without a lock, the build goes ahead: a stuck builder costs time, never a verdict."""
|
|
1828
|
+
descriptor = None
|
|
1829
|
+
try:
|
|
1830
|
+
lock = index_path(root).with_suffix(".lock")
|
|
1831
|
+
lock.parent.mkdir(parents=True, exist_ok=True, mode=0o700)
|
|
1832
|
+
descriptor = os.open(lock, os.O_RDWR | os.O_CREAT | os.O_NOFOLLOW, 0o600)
|
|
1833
|
+
deadline = time.monotonic() + wait
|
|
1834
|
+
while True:
|
|
1835
|
+
try:
|
|
1836
|
+
fcntl.flock(descriptor, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
1837
|
+
break
|
|
1838
|
+
except BlockingIOError:
|
|
1839
|
+
if time.monotonic() >= deadline:
|
|
1840
|
+
break
|
|
1841
|
+
time.sleep(0.05)
|
|
1842
|
+
except OSError:
|
|
1843
|
+
pass
|
|
1844
|
+
try:
|
|
1845
|
+
yield
|
|
1846
|
+
finally:
|
|
1847
|
+
if descriptor is not None:
|
|
1848
|
+
os.close(descriptor)
|
|
1849
|
+
|
|
1850
|
+
|
|
1851
|
+
def tree(memory_root: Path, release: Iterable[str] = (), *, trust_cache: bool = False, fresh: bool = False,
|
|
1852
|
+
build: bool = True) -> Tree | None:
|
|
1853
|
+
"""Memory as walked within FRESH_SECONDS (now, when ``fresh``); built again only when a document's
|
|
1854
|
+
size, times, inode or target changed. With ``trust_cache`` (the operator's own reads and slot) a walk this
|
|
1855
|
+
user already built is read back from the verdict index, and builds of one memory take turns, so one memory
|
|
1856
|
+
state is built once per machine. ``build=False`` returns None instead of building."""
|
|
1857
|
+
return current(memory_root, release, trust_cache=trust_cache, fresh=fresh, build=build)[0]
|
|
1858
|
+
|
|
1859
|
+
|
|
1860
|
+
def current(memory_root: Path, release: Iterable[str] = (), *, trust_cache: bool = False, fresh: bool = False,
|
|
1861
|
+
build: bool = True) -> tuple[Tree | None, str]:
|
|
1862
|
+
"""``tree``, and the digest of the walk it answers for: the walk in effect now."""
|
|
1863
|
+
root, release = _resolved(memory_root), _prefixes(release)
|
|
1864
|
+
key = ("tree", str(root), release, trust_cache)
|
|
1865
|
+
for seen in (_TREES.get(key), None if build else _TREES.get(("unbuilt", *key))):
|
|
1866
|
+
if seen is not None and not fresh and time.monotonic() - seen[0] < FRESH_SECONDS:
|
|
1867
|
+
return seen[2], seen[1]
|
|
1868
|
+
walked, listed = time.monotonic(), _now()
|
|
1869
|
+
listing = sorted((rel, target, info.st_size, info.st_mtime_ns, info.st_ctime_ns, info.st_ino)
|
|
1870
|
+
for rel, target, info in _documents(root))
|
|
1871
|
+
digest = hashlib.sha256(json.dumps([INDEX_VERSION, HELD_RULES_VERSION, release, listing]).encode()).hexdigest()
|
|
1872
|
+
|
|
1873
|
+
def documents():
|
|
1874
|
+
for rel, target, *_ in listing:
|
|
1875
|
+
try:
|
|
1876
|
+
yield rel, target, (root / target).read_bytes()
|
|
1877
|
+
except OSError:
|
|
1878
|
+
continue
|
|
1879
|
+
|
|
1880
|
+
def full() -> Tree:
|
|
1881
|
+
return Tree(documents(), release=release, cache=cache_path(root) if trust_cache else None)
|
|
1882
|
+
|
|
1883
|
+
def make() -> Tree | None:
|
|
1884
|
+
if not trust_cache:
|
|
1885
|
+
found = full() if build else None
|
|
1886
|
+
elif (found := _read_index(root, digest, listing, listed, full, release)) is None and build:
|
|
1887
|
+
with _turn(root):
|
|
1888
|
+
found = _read_index(root, digest, listing, listed, full, release) # built meanwhile by the turn before
|
|
1889
|
+
if found is None:
|
|
1890
|
+
found = full()
|
|
1891
|
+
if found.documents == len(listing): # one it could not read leaves no index behind
|
|
1892
|
+
_write_index(root, digest, listed, found)
|
|
1893
|
+
if found is not None:
|
|
1894
|
+
found.renew = lambda: tree(root, release, trust_cache=trust_cache, fresh=True)
|
|
1895
|
+
found.walk = digest
|
|
1896
|
+
found._judged = _Judged(_judged_path(root) if trust_cache else None, digest)
|
|
1897
|
+
return found
|
|
1898
|
+
|
|
1899
|
+
found = _remember(key, digest, make, walked)
|
|
1900
|
+
if found is None: # nothing to read back for this walk: calls that would not build reuse it
|
|
1901
|
+
_TREES[("unbuilt", *key)] = (walked, digest, None)
|
|
1902
|
+
return found, digest
|
|
1903
|
+
|
|
1904
|
+
|
|
1905
|
+
LINK_HOPS = 40 # the kernel's limit: a longer chain, or a cycle, resolves to nothing
|
|
1906
|
+
|
|
1907
|
+
|
|
1908
|
+
def committed_target(memory, name: str) -> str | None:
|
|
1909
|
+
"""The committed regular file ``name`` reads as, resolved part by part as the file system resolves a path: a
|
|
1910
|
+
link anywhere along it (a chain, a folder link, a target in a dot folder or under another name) is followed.
|
|
1911
|
+
None for a cycle, an absolute target, a target above memory, or anything but a committed file at the end."""
|
|
1912
|
+
resolved, pending, hops = [], name.split("/"), 0
|
|
1913
|
+
budget = getattr(memory, "budget", None)
|
|
1914
|
+
while pending:
|
|
1915
|
+
if budget is not None:
|
|
1916
|
+
budget.remaining()
|
|
1917
|
+
budget.charge("work", 1 + len(pending[0]))
|
|
1918
|
+
part = pending.pop(0)
|
|
1919
|
+
if part in ("", "."):
|
|
1920
|
+
continue
|
|
1921
|
+
if part == "..":
|
|
1922
|
+
if not resolved:
|
|
1923
|
+
return None
|
|
1924
|
+
resolved.pop()
|
|
1925
|
+
continue
|
|
1926
|
+
resolved.append(part)
|
|
1927
|
+
entry = memory.entries.get("/".join(resolved))
|
|
1928
|
+
if entry is not None and entry[0] == "120000":
|
|
1929
|
+
hops += 1
|
|
1930
|
+
link = memory.read("/".join(resolved)).decode("utf-8", "surrogateescape")
|
|
1931
|
+
if hops > (budget.limits.link_hops if budget is not None else LINK_HOPS) or not link or link.startswith("/"):
|
|
1932
|
+
return None
|
|
1933
|
+
resolved.pop()
|
|
1934
|
+
pending = link.split("/") + pending
|
|
1935
|
+
entry = memory.entries.get("/".join(resolved))
|
|
1936
|
+
return "/".join(resolved) if entry is not None and entry[0] in {"100644", "100755"} else None
|
|
1937
|
+
|
|
1938
|
+
|
|
1939
|
+
def committed_tree(memory, release: Iterable[str] = ()) -> Tree:
|
|
1940
|
+
"""Memory as committed at one revision (an ``access.CommittedMemory``), never from a cache. A committed link
|
|
1941
|
+
reads as the file it resolves to, as the walk reads one on disk."""
|
|
1942
|
+
release = _prefixes(release)
|
|
1943
|
+
|
|
1944
|
+
def build() -> Tree:
|
|
1945
|
+
names = {name: mode for name, (mode, _oid) in memory.entries.items() if mode in {"100644", "100755", "120000"}
|
|
1946
|
+
and name.lower().endswith(".md") and not any(part.startswith(".") for part in name.split("/"))}
|
|
1947
|
+
memory.prefetch([name for name, (mode, _oid) in memory.entries.items() if mode == "120000"])
|
|
1948
|
+
found = {name: name if mode != "120000" else committed_target(memory, name) for name, mode in names.items()}
|
|
1949
|
+
found = {name: target for name, target in found.items() if target}
|
|
1950
|
+
memory.prefetch(sorted(set(found.values())))
|
|
1951
|
+
return Tree(((name, target, memory.read(target)) for name, target in sorted(found.items())), release=release)
|
|
1952
|
+
|
|
1953
|
+
return _remember(("committed", str(Path(memory.root).resolve()), release), memory.revision, build, time.monotonic())
|
|
1954
|
+
|
|
1955
|
+
|
|
1956
|
+
def _file_identity(info):
|
|
1957
|
+
return (info.st_dev, info.st_ino, info.st_mode, info.st_nlink, info.st_size, info.st_mtime_ns, info.st_ctime_ns)
|
|
1958
|
+
|
|
1959
|
+
|
|
1960
|
+
def _inventory_row(budget, rows, *row):
|
|
1961
|
+
data = json.dumps(row, ensure_ascii=True, separators=(",", ":"))
|
|
1962
|
+
budget.charge("inventory_bytes", len(data.encode()))
|
|
1963
|
+
rows.append(row)
|
|
1964
|
+
|
|
1965
|
+
|
|
1966
|
+
def _bounded_target(root_fd, relative, budget, rows):
|
|
1967
|
+
"""Read one spelling through owned directory descriptors; never follow an escape."""
|
|
1968
|
+
pending, resolved, directories, hops = relative.split("/"), [], [os.dup(root_fd)], 0
|
|
1969
|
+
try:
|
|
1970
|
+
while pending:
|
|
1971
|
+
budget.remaining()
|
|
1972
|
+
part = pending.pop(0)
|
|
1973
|
+
if part in ("", "."):
|
|
1974
|
+
continue
|
|
1975
|
+
if part == "..":
|
|
1976
|
+
if not resolved:
|
|
1977
|
+
return None
|
|
1978
|
+
resolved.pop(); os.close(directories.pop())
|
|
1979
|
+
continue
|
|
1980
|
+
named = "/".join([*resolved, part])
|
|
1981
|
+
if len(os.fsencode(named)) > budget.limits.path_bytes:
|
|
1982
|
+
budget.fail()
|
|
1983
|
+
try:
|
|
1984
|
+
info = os.stat(part, dir_fd=directories[-1], follow_symlinks=False)
|
|
1985
|
+
except FileNotFoundError:
|
|
1986
|
+
_inventory_row(budget, rows, "absent-target", named)
|
|
1987
|
+
return None
|
|
1988
|
+
_inventory_row(budget, rows, "target", named, _file_identity(info))
|
|
1989
|
+
if stat.S_ISLNK(info.st_mode):
|
|
1990
|
+
hops += 1
|
|
1991
|
+
if hops > budget.limits.link_hops:
|
|
1992
|
+
return None
|
|
1993
|
+
link = os.readlink(part, dir_fd=directories[-1])
|
|
1994
|
+
if len(os.fsencode(link)) > budget.limits.path_bytes:
|
|
1995
|
+
budget.fail()
|
|
1996
|
+
_inventory_row(budget, rows, "link", named, link)
|
|
1997
|
+
if _file_identity(os.stat(part, dir_fd=directories[-1], follow_symlinks=False)) != _file_identity(info):
|
|
1998
|
+
budget.fail()
|
|
1999
|
+
if not link or link.startswith("/"):
|
|
2000
|
+
return None
|
|
2001
|
+
pending = link.split("/") + pending
|
|
2002
|
+
continue
|
|
2003
|
+
if stat.S_ISDIR(info.st_mode):
|
|
2004
|
+
if not pending:
|
|
2005
|
+
return None
|
|
2006
|
+
descriptor = os.open(part, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=directories[-1])
|
|
2007
|
+
if _file_identity(os.fstat(descriptor)) != _file_identity(info):
|
|
2008
|
+
os.close(descriptor); budget.fail()
|
|
2009
|
+
resolved.append(part); directories.append(descriptor)
|
|
2010
|
+
continue
|
|
2011
|
+
if pending or not stat.S_ISREG(info.st_mode):
|
|
2012
|
+
return None
|
|
2013
|
+
if not info.st_mode & 0o444 or info.st_size > budget.limits.dependency_bytes:
|
|
2014
|
+
budget.fail()
|
|
2015
|
+
descriptor = os.open(part, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK, dir_fd=directories[-1])
|
|
2016
|
+
with os.fdopen(descriptor, "rb", buffering=0) as stream:
|
|
2017
|
+
before = os.fstat(stream.fileno())
|
|
2018
|
+
if _file_identity(before) != _file_identity(info):
|
|
2019
|
+
budget.fail()
|
|
2020
|
+
budget.charge("total_bytes", before.st_size)
|
|
2021
|
+
data = stream.read(before.st_size + 1)
|
|
2022
|
+
budget.remaining()
|
|
2023
|
+
if len(data) != before.st_size or _file_identity(os.fstat(stream.fileno())) != _file_identity(before):
|
|
2024
|
+
budget.fail()
|
|
2025
|
+
return named, data, _file_identity(info)
|
|
2026
|
+
return None
|
|
2027
|
+
finally:
|
|
2028
|
+
for descriptor in directories:
|
|
2029
|
+
os.close(descriptor)
|
|
2030
|
+
|
|
2031
|
+
|
|
2032
|
+
def _bounded_working_source(root, budget):
|
|
2033
|
+
budget = _read_budget(budget)
|
|
2034
|
+
root = Path(root)
|
|
2035
|
+
if not root.is_absolute() or root != root.resolve(strict=True):
|
|
2036
|
+
budget.fail()
|
|
2037
|
+
root_fd = os.open(root, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW)
|
|
2038
|
+
rows, documents, identities = [], {}, {}
|
|
2039
|
+
root_identity = _file_identity(os.fstat(root_fd))
|
|
2040
|
+
|
|
2041
|
+
def walk(directory, prefix):
|
|
2042
|
+
budget.remaining()
|
|
2043
|
+
before = os.fstat(directory)
|
|
2044
|
+
if not before.st_mode & 0o444 or not before.st_mode & 0o111:
|
|
2045
|
+
budget.fail()
|
|
2046
|
+
_inventory_row(budget, rows, "directory", prefix, _file_identity(before))
|
|
2047
|
+
names = []
|
|
2048
|
+
with os.scandir(directory) as entries:
|
|
2049
|
+
for entry in entries:
|
|
2050
|
+
budget.charge("entries")
|
|
2051
|
+
name = entry.name
|
|
2052
|
+
if len(os.fsencode(prefix + name)) > budget.limits.path_bytes:
|
|
2053
|
+
budget.fail()
|
|
2054
|
+
names.append(name)
|
|
2055
|
+
for name in sorted(names):
|
|
2056
|
+
budget.remaining()
|
|
2057
|
+
relative = prefix + name
|
|
2058
|
+
if name.startswith("."):
|
|
2059
|
+
_inventory_row(budget, rows, "hidden", relative)
|
|
2060
|
+
continue
|
|
2061
|
+
info = os.stat(name, dir_fd=directory, follow_symlinks=False)
|
|
2062
|
+
_inventory_row(budget, rows, "entry", relative, _file_identity(info))
|
|
2063
|
+
if stat.S_ISDIR(info.st_mode):
|
|
2064
|
+
child = os.open(name, os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW, dir_fd=directory)
|
|
2065
|
+
try:
|
|
2066
|
+
if _file_identity(os.fstat(child)) != _file_identity(info):
|
|
2067
|
+
budget.fail()
|
|
2068
|
+
walk(child, relative + "/")
|
|
2069
|
+
finally:
|
|
2070
|
+
os.close(child)
|
|
2071
|
+
elif name.lower().endswith(".md") and stat.S_IFMT(info.st_mode) in {stat.S_IFREG, stat.S_IFLNK}:
|
|
2072
|
+
if (selected := _bounded_target(root_fd, relative, budget, rows)) is not None:
|
|
2073
|
+
if len(documents) >= budget.limits.documents:
|
|
2074
|
+
budget.fail()
|
|
2075
|
+
target, data, identity = selected
|
|
2076
|
+
documents[relative] = (target, data)
|
|
2077
|
+
identities[relative] = (_file_identity(info), identity)
|
|
2078
|
+
_inventory_row(budget, rows, "content", relative, target, hashlib.sha256(data).hexdigest())
|
|
2079
|
+
if _file_identity(os.fstat(directory)) != _file_identity(before):
|
|
2080
|
+
budget.fail()
|
|
2081
|
+
|
|
2082
|
+
try:
|
|
2083
|
+
walk(root_fd, "")
|
|
2084
|
+
budget.remaining()
|
|
2085
|
+
fingerprint = hashlib.sha256(json.dumps(rows, ensure_ascii=True, separators=(",", ":")).encode()).hexdigest()
|
|
2086
|
+
return root_identity, fingerprint, documents, identities
|
|
2087
|
+
except (OSError, ValueError, RuntimeError):
|
|
2088
|
+
budget.fail()
|
|
2089
|
+
finally:
|
|
2090
|
+
os.close(root_fd)
|
|
2091
|
+
|
|
2092
|
+
|
|
2093
|
+
class _BoundedTree:
|
|
2094
|
+
__slots__ = ("root", "budget", "release", "rules", "source", "identity", "fingerprint", "documents", "identities", "tree")
|
|
2095
|
+
|
|
2096
|
+
def __init__(self, root, release, budget, committed):
|
|
2097
|
+
self.root, self.budget, self.release, self.rules = Path(root), budget, tuple(release), HELD_RULES_VERSION
|
|
2098
|
+
self.source = committed
|
|
2099
|
+
if committed is None:
|
|
2100
|
+
self.identity, self.fingerprint, self.documents, self.identities = _bounded_working_source(self.root, budget)
|
|
2101
|
+
else:
|
|
2102
|
+
self.identity, self.fingerprint, self.documents, self.identities = committed.hold_documents()
|
|
2103
|
+
token = _WORK_BUDGET.set(budget)
|
|
2104
|
+
try:
|
|
2105
|
+
self.tree = Tree(((rel, target, raw) for rel, (target, raw) in self.documents.items()), release=release)
|
|
2106
|
+
budget.remaining()
|
|
2107
|
+
finally:
|
|
2108
|
+
_WORK_BUDGET.reset(token)
|
|
2109
|
+
|
|
2110
|
+
def classes(self, relative, raw, target=None):
|
|
2111
|
+
self.budget.remaining()
|
|
2112
|
+
entry = self.documents.get(relative)
|
|
2113
|
+
if self.rules != HELD_RULES_VERSION or entry is None or entry != (target or relative, raw):
|
|
2114
|
+
self.budget.fail()
|
|
2115
|
+
# Exact captured entries only. The ordinary Tree renewal/derivation path
|
|
2116
|
+
# is deliberately unreachable, including for a caller's replacement bytes.
|
|
2117
|
+
if admin_class(raw):
|
|
2118
|
+
return (1,)
|
|
2119
|
+
found = self.tree.verdict(relative, hashlib.sha256(raw).hexdigest(), entry[0])
|
|
2120
|
+
if found is None:
|
|
2121
|
+
self.budget.fail()
|
|
2122
|
+
return (RULE_CLASS[found],) if found in RULE_CLASS else ()
|
|
2123
|
+
|
|
2124
|
+
def recheck(self):
|
|
2125
|
+
self.budget.remaining()
|
|
2126
|
+
if self.rules != HELD_RULES_VERSION:
|
|
2127
|
+
self.budget.fail()
|
|
2128
|
+
if self.source is not None:
|
|
2129
|
+
self.source.recheck()
|
|
2130
|
+
else:
|
|
2131
|
+
identity, fingerprint, _docs, _identities = _bounded_working_source(self.root, self.budget)
|
|
2132
|
+
if identity != self.identity or fingerprint != self.fingerprint:
|
|
2133
|
+
self.budget.fail()
|
|
2134
|
+
|
|
2135
|
+
|
|
2136
|
+
def bounded_tree(memory_root, release=(), *, budget, committed=None):
|
|
2137
|
+
"""Complete bounded rule inputs; no cache, lock, shadow, renewal or writes."""
|
|
2138
|
+
budget = _read_budget(budget)
|
|
2139
|
+
try:
|
|
2140
|
+
if committed is not None:
|
|
2141
|
+
from .access import _BoundedCommittedMemory
|
|
2142
|
+
if type(committed) is not _BoundedCommittedMemory or committed.budget is not budget or committed.root != Path(memory_root):
|
|
2143
|
+
budget.fail()
|
|
2144
|
+
return _BoundedTree(memory_root, release, budget, committed)
|
|
2145
|
+
except (OSError, ValueError, RuntimeError):
|
|
2146
|
+
budget.fail()
|
|
2147
|
+
|
|
2148
|
+
|
|
2149
|
+
def verdicts(memory_root: Path, *, admins: Iterable[str] = (), release: Iterable[str] = (),
|
|
2150
|
+
trust_cache: bool = False, cache: Path | None = None) -> Verdicts:
|
|
2151
|
+
"""Classify every Markdown document under ``memory_root``.
|
|
2152
|
+
|
|
2153
|
+
Class 1 is read from each file on every call. A rule verdict is cached under (path as named,
|
|
2154
|
+
resolved path, content sha256, rules version, admin-set digest, release digest), a derivative's
|
|
2155
|
+
under the digest of the admin-only set and its names instead of the last two; the cache is read
|
|
2156
|
+
only with ``trust_cache=True``: the operator's own reads and slot. ``release`` (memory paths)
|
|
2157
|
+
lifts a rule verdict, never ``admin: true``. A symlink inside memory is held when either its own
|
|
2158
|
+
path or its target is, as the predicate checks both spellings.
|
|
2159
|
+
"""
|
|
2160
|
+
root = Path(memory_root).resolve()
|
|
2161
|
+
|
|
2162
|
+
def documents():
|
|
2163
|
+
for rel, target, _info in _documents(root):
|
|
2164
|
+
try:
|
|
2165
|
+
yield rel, target, (root / target).read_bytes()
|
|
2166
|
+
except OSError:
|
|
2167
|
+
continue
|
|
2168
|
+
|
|
2169
|
+
found = Tree(documents(), release=release, cache=cache if trust_cache else None)
|
|
2170
|
+
return Verdicts(found.held, found.computed, found.documents, holds_epoch(admins, held=found.held))
|
|
2171
|
+
|
|
2172
|
+
|
|
2173
|
+
def holds_epoch(admins: Iterable[str], released: Iterable[str] = (), held: Mapping[str, str] | None = None) -> str:
|
|
2174
|
+
"""Digest the rules version, the admin set, the releases and, when known, the held set (path -> class)."""
|
|
2175
|
+
payload: list = [HELD_RULES_VERSION, sorted({handle.strip().casefold() for handle in admins})]
|
|
2176
|
+
payload += [sorted(set(released))] if released else []
|
|
2177
|
+
if held is not None:
|
|
2178
|
+
payload.append(sorted(held.items()))
|
|
2179
|
+
return hashlib.sha256(json.dumps(payload, separators=(",", ":")).encode()).hexdigest()
|
|
2180
|
+
|
|
2181
|
+
|
|
2182
|
+
# -- Redaction: held names leave no text a non-admin is handed --------------------------------------
|
|
2183
|
+
class RedactionError(EgregoreRuntimeError):
|
|
2184
|
+
"""A held name survived redaction, so nothing is handed over."""
|
|
2185
|
+
|
|
2186
|
+
|
|
2187
|
+
_RUN_STOPS = frozenset("`'\"()[]<>{}|,;*~\u2018\u2019\u201c\u201d")
|
|
2188
|
+
_RUN = _lazy(r"[^\s`'\"()\[\]<>{}|,;*~\u2018\u2019\u201c\u201d]+")
|
|
2189
|
+
_WORD = _lazy(r"[^\W_]+")
|
|
2190
|
+
# A run's segments split at / : = # ? & and trimmed of _ . ! at both ends: each name's last one is one.
|
|
2191
|
+
_D = "\\s`'\"()\\[\\]<>{}|,;*~\u2018\u2019\u201c\u201d/:=#?&"
|
|
2192
|
+
_PIECE = _lazy(r"[:=#?&]")
|
|
2193
|
+
_SEGMENT = _lazy(f"[^{_D}_.!](?:[^{_D}]*[^{_D}_.!])?")
|
|
2194
|
+
_ESCAPE = _lazy(r"%[0-9A-Fa-f]{2}")
|
|
2195
|
+
|
|
2196
|
+
|
|
2197
|
+
def _run(text: str, start: int, end: int) -> tuple[int, int]:
|
|
2198
|
+
"""The whole run (see ``_RUN``) around a segment."""
|
|
2199
|
+
while start and not (text[start - 1].isspace() or text[start - 1] in _RUN_STOPS):
|
|
2200
|
+
start -= 1
|
|
2201
|
+
while end < len(text) and not (text[end].isspace() or text[end] in _RUN_STOPS):
|
|
2202
|
+
end += 1
|
|
2203
|
+
return start, end
|
|
2204
|
+
|
|
2205
|
+
|
|
2206
|
+
def _title_words(title: str) -> tuple[str, ...]:
|
|
2207
|
+
return tuple(word.casefold() for word in _WORD.findall(title))
|
|
2208
|
+
|
|
2209
|
+
|
|
2210
|
+
def _spelled(names) -> re.Pattern | None:
|
|
2211
|
+
"""Exact spellings, case aside, never inside a word or a dotted or hyphenated name; an
|
|
2212
|
+
underscore is a boundary, as Markdown emphasis wraps a path in it."""
|
|
2213
|
+
names = "|".join(map(re.escape, sorted(names, key=len, reverse=True)))
|
|
2214
|
+
return re.compile(rf"(?<![^\W_])(?<![.-])(?:{names})(?![^\W_])(?!-)", re.IGNORECASE) if names else None
|
|
2215
|
+
|
|
2216
|
+
|
|
2217
|
+
def _strip(text: str, spans: list[tuple[int, int]]) -> str:
|
|
2218
|
+
kept, at = [], 0
|
|
2219
|
+
for start, end in spans:
|
|
2220
|
+
if start >= at:
|
|
2221
|
+
kept.append(text[at:start])
|
|
2222
|
+
at = max(at, end)
|
|
2223
|
+
return "".join(kept) + text[at:]
|
|
2224
|
+
|
|
2225
|
+
|
|
2226
|
+
def describe(canonical: str, data: bytes) -> tuple[str, str, tuple[str, ...]]:
|
|
2227
|
+
"""A document's title, its stable artifact id, and every id, slug or session id it declares."""
|
|
2228
|
+
text = data.decode("utf-8", "replace")
|
|
2229
|
+
try:
|
|
2230
|
+
fields, body = split_frontmatter(text)
|
|
2231
|
+
except ArtifactSchemaError:
|
|
2232
|
+
fields, body = {}, text
|
|
2233
|
+
title = next((str(fields[key]).strip() for key in ("title", "topic", "name") if fields.get(key)), "")
|
|
2234
|
+
if not title and (heading := re.search(r"^#\s+(.+?)\s*$", body, re.MULTILINE)):
|
|
2235
|
+
title = heading.group(1).strip()
|
|
2236
|
+
aliases = tuple(str(fields[key]).strip() for key in ("artifact_id", "id", "session_id", "slug")
|
|
2237
|
+
if isinstance(fields.get(key), (str, int)))
|
|
2238
|
+
return title, canonical_artifact_id(text, canonical), aliases
|
|
2239
|
+
|
|
2240
|
+
|
|
2241
|
+
class Redactor:
|
|
2242
|
+
"""Spellings that name a held document, except those a shown document also answers to
|
|
2243
|
+
(``AdminReferences`` precedent): a shared file name narrows to its shortest folder path no
|
|
2244
|
+
shown path spells; a shared slug or title is ordinary vocabulary. An id is an identity: a held
|
|
2245
|
+
one stays held though a shown document declares it too or is named for it. A name no run can
|
|
2246
|
+
hold (a space or a quote in it) is matched by its exact spelling: a held document is found by
|
|
2247
|
+
its path, never by a name a shown one shares.
|
|
2248
|
+
|
|
2249
|
+
``held_docs``: (canonical path, title, artifact id, declared ids) per held document;
|
|
2250
|
+
``shown_titles``: the shown documents' titles, or a function that reads them on first need
|
|
2251
|
+
(a held title found in a text)."""
|
|
2252
|
+
|
|
2253
|
+
def __init__(self, held_docs: Iterable[tuple[str, str, str, tuple[str, ...]]], shown_paths: Iterable[str],
|
|
2254
|
+
shown_titles: Iterable[str] | Callable[[], Iterable[str]]):
|
|
2255
|
+
shown_rel = [path.removeprefix("memory/").casefold() for path in shown_paths]
|
|
2256
|
+
stems = {rel.rsplit("/", 1)[-1].removesuffix(".md") for rel in shown_rel}
|
|
2257
|
+
stems |= {DATE_PREFIX.sub("", stem) for stem in stems}
|
|
2258
|
+
listed = "\n".join(shown_rel)
|
|
2259
|
+
|
|
2260
|
+
def shared(name: str) -> bool: # a shown path answers to this spelling too
|
|
2261
|
+
if _RUN.fullmatch(name):
|
|
2262
|
+
return any(other == name or other.endswith("/" + name) for other in shown_rel)
|
|
2263
|
+
return _spelled([name]).search(listed) is not None
|
|
2264
|
+
self.names: set[str] = set()
|
|
2265
|
+
self._titles: dict[str, set[tuple[str, ...]]] = {}
|
|
2266
|
+
self._shown_titles, self._kept, self._lasts = shown_titles, None, None
|
|
2267
|
+
for canonical, title, artifact_id, aliases in held_docs:
|
|
2268
|
+
rel = canonical.removeprefix("memory/").casefold()
|
|
2269
|
+
self.names.update(n for n in ("memory/" + rel, rel, "memory/" + rel.removesuffix(".md"),
|
|
2270
|
+
rel.removesuffix(".md")) if "/" in n)
|
|
2271
|
+
parts = rel.split("/")
|
|
2272
|
+
tails = ["/".join(parts[-k:]) for k in range(1, len(parts) + 1)]
|
|
2273
|
+
if shared(tails[0]):
|
|
2274
|
+
self.names.update(tail for tail in tails[1:] if not shared(tail))
|
|
2275
|
+
else:
|
|
2276
|
+
self.names.add(tails[0])
|
|
2277
|
+
stem = parts[-1].removesuffix(".md")
|
|
2278
|
+
self.names.update(n for n in (stem, DATE_PREFIX.sub("", stem)) if distinctive_slug(n) and n not in stems)
|
|
2279
|
+
ident = artifact_id.casefold() # its identity: a shown file named for it shares nothing
|
|
2280
|
+
self.names.update(n for n in {ident, *(a.casefold() for a in aliases)}
|
|
2281
|
+
if (len(n) >= 8 or distinctive_slug(n) or not n.isalpha()) and (n == ident or n not in stems))
|
|
2282
|
+
words = _title_words(title)
|
|
2283
|
+
if len(words) >= 3 and len(" ".join(words)) >= 20:
|
|
2284
|
+
self._titles.setdefault(words[0], set()).add(words)
|
|
2285
|
+
self.literals = {name for name in self.names if not _RUN.fullmatch(name) and not shared(name)}
|
|
2286
|
+
self.literal = _spelled(self.literals)
|
|
2287
|
+
|
|
2288
|
+
@property
|
|
2289
|
+
def lasts(self) -> set[str]:
|
|
2290
|
+
if self._lasts is None:
|
|
2291
|
+
self._lasts = {re.split(r"[/:=#?&]", name)[-1].strip("_.!") for name in self.names} - {""}
|
|
2292
|
+
return self._lasts
|
|
2293
|
+
|
|
2294
|
+
def titles(self) -> set[tuple[str, ...]]:
|
|
2295
|
+
"""The held titles of three or more words and 20 or more characters that no shown document carries."""
|
|
2296
|
+
if self._kept is None:
|
|
2297
|
+
shown = self._shown_titles() if callable(self._shown_titles) else self._shown_titles
|
|
2298
|
+
shown = {_title_words(title) for title in shown}
|
|
2299
|
+
self._kept = {title for group in self._titles.values() for title in group if title not in shown}
|
|
2300
|
+
return self._kept
|
|
2301
|
+
|
|
2302
|
+
def revision(self) -> list:
|
|
2303
|
+
return [sorted(self.names), sorted(self.titles()), sorted(self.literals)]
|
|
2304
|
+
|
|
2305
|
+
def _named(self, run: str) -> bool:
|
|
2306
|
+
value = unquote(run).casefold()
|
|
2307
|
+
for piece in (value, *_PIECE.split(value)): # a whole id, then each query, fragment or label part
|
|
2308
|
+
parts = piece.strip("_.!?/").split("/")
|
|
2309
|
+
if any("/".join(parts[i:]) in self.names for i in range(len(parts))):
|
|
2310
|
+
return True
|
|
2311
|
+
return False
|
|
2312
|
+
|
|
2313
|
+
def spans(self, text: str) -> list[tuple[int, int]]:
|
|
2314
|
+
# Set tests in C first: most texts hold no held name's last segment and no whole title.
|
|
2315
|
+
# When casefolding kept every offset, the folded text is searched and its spans used as is.
|
|
2316
|
+
folded, found = text.casefold(), set()
|
|
2317
|
+
aligned = folded if len(folded) == len(text) else None
|
|
2318
|
+
if not self.lasts.isdisjoint(_SEGMENT.findall(folded)):
|
|
2319
|
+
found.update(_run(text, m.start(), m.end()) for m in _SEGMENT.finditer(aligned or text)
|
|
2320
|
+
if m.group().casefold() in self.lasts)
|
|
2321
|
+
if _ESCAPE.search(text):
|
|
2322
|
+
found.update(m.span() for m in _RUN.finditer(text) if "%" in m.group())
|
|
2323
|
+
found = {(start, end) for start, end in found if self._named(text[start:end])}
|
|
2324
|
+
if self.literal is not None:
|
|
2325
|
+
found.update(_run(text, m.start(), m.end()) for m in self.literal.finditer(text))
|
|
2326
|
+
present = set(_WORD.findall(folded)) if self._titles else set()
|
|
2327
|
+
for title in {t for first in present & self._titles.keys() for t in self._titles[first] if present.issuperset(t)}:
|
|
2328
|
+
if aligned is not None:
|
|
2329
|
+
pattern = r"(?<![^\W_])" + r"[\W_]+".join(map(re.escape, title)) + r"(?![^\W_])"
|
|
2330
|
+
spans = {m.span() for m in re.finditer(pattern, aligned)}
|
|
2331
|
+
else:
|
|
2332
|
+
words = list(_WORD.finditer(text))
|
|
2333
|
+
folded_words = [w.group().casefold() for w in words]
|
|
2334
|
+
spans = {(words[i].start(), words[i + len(title) - 1].end()) for i in range(len(words))
|
|
2335
|
+
if tuple(folded_words[i:i + len(title)]) == title}
|
|
2336
|
+
if spans and title in self.titles(): # a title a shown document carries too is ordinary words
|
|
2337
|
+
found.update(spans)
|
|
2338
|
+
return sorted(found)
|
|
2339
|
+
|
|
2340
|
+
def redact(self, text: str) -> str:
|
|
2341
|
+
"""Strip every held name; the last pass finding none is the leak check, else it raises."""
|
|
2342
|
+
for _ in range(5):
|
|
2343
|
+
spans = self.spans(text)
|
|
2344
|
+
if not spans:
|
|
2345
|
+
return text
|
|
2346
|
+
text = _strip(text, spans)
|
|
2347
|
+
raise RedactionError("leak check failed: a held name survived redaction")
|
|
2348
|
+
|
|
2349
|
+
def scrub(self, value: Any, keys: bool = True) -> Any:
|
|
2350
|
+
"""Every string in a value redacted: nested lists, tuples and dicts, their keys too unless ``keys`` is
|
|
2351
|
+
False (a caller's own field names)."""
|
|
2352
|
+
if isinstance(value, str):
|
|
2353
|
+
return self.redact(value)
|
|
2354
|
+
if isinstance(value, (list, tuple)):
|
|
2355
|
+
return type(value)(self.scrub(item, keys) for item in value)
|
|
2356
|
+
if isinstance(value, dict):
|
|
2357
|
+
return {self.scrub(key) if keys else key: self.scrub(item, keys) for key, item in value.items()}
|
|
2358
|
+
if is_dataclass(value) and not isinstance(value, type):
|
|
2359
|
+
return replace(value, **{field.name: self.scrub(getattr(value, field.name), keys) for field in fields(value)})
|
|
2360
|
+
return value
|
|
2361
|
+
|
|
2362
|
+
|
|
2363
|
+
def redact(decision, item: Any, *, root: Path, keys: bool = False) -> Any:
|
|
2364
|
+
"""``item`` with every held name a decision strips taken out of each string it holds (``Redactor.scrub``;
|
|
2365
|
+
dict keys are the caller's field names and stay unless ``keys``); unchanged for an admin and for a read that holds class 1
|
|
2366
|
+
alone. Raises ``RedactionError`` when a name survives or the held set cannot be read. ``root`` is the memory
|
|
2367
|
+
root the decision's checks use."""
|
|
2368
|
+
from .policy import redactor # the decision's held set, judged where every class is
|
|
2369
|
+
try:
|
|
2370
|
+
found = redactor(decision, root)
|
|
2371
|
+
except (OSError, ValueError, subprocess.SubprocessError) as error:
|
|
2372
|
+
raise RedactionError("the held names could not be read, so nothing is handed over") from error
|
|
2373
|
+
return item if found is None else found.scrub(item, keys=keys)
|
|
2374
|
+
|
|
2375
|
+
|
|
2376
|
+
def identity(decision, artifact_id: str, path: str, *, root: Path) -> str:
|
|
2377
|
+
"""An item's id as handed over: its own, or the id its path gives when it spells a held name (the
|
|
2378
|
+
projection's manifest rule)."""
|
|
2379
|
+
if redact(decision, artifact_id, root=root) == artifact_id:
|
|
2380
|
+
return artifact_id
|
|
2381
|
+
try:
|
|
2382
|
+
return artifact_id_from_path(path.removeprefix("memory/"))
|
|
2383
|
+
except ArtifactSchemaError:
|
|
2384
|
+
return ""
|
|
2385
|
+
|
|
2386
|
+
|
|
2387
|
+
def package(decision, item: Any, *, root: Path, paths: Iterable[str] = ()) -> Any | None:
|
|
2388
|
+
"""How memory text leaves the Runtime: ``allows`` on every path the item names, then ``redact`` on
|
|
2389
|
+
every string. None when a path is refused or the redaction cannot be completed."""
|
|
2390
|
+
from .policy import allows
|
|
2391
|
+
try:
|
|
2392
|
+
if any(not allows(decision, path, "read", root=root) for path in paths):
|
|
2393
|
+
return None
|
|
2394
|
+
return redact(decision, item, root=root)
|
|
2395
|
+
except RedactionError:
|
|
2396
|
+
return None
|
|
2397
|
+
|
|
2398
|
+
|
|
2399
|
+
def kept(decision, rows: Iterable, *, root: Path, path: str | None = None) -> list:
|
|
2400
|
+
"""Each row ``package``d on its own, under the memory path in its ``path`` field: a refused row is dropped."""
|
|
2401
|
+
def named(row) -> tuple[str, ...]:
|
|
2402
|
+
value = row.get(path) if path and isinstance(row, Mapping) else None
|
|
2403
|
+
return ("memory/" + value.removeprefix("memory/"),) if isinstance(value, str) and value else ()
|
|
2404
|
+
return [item for item in (package(decision, row, root=root, paths=named(row)) for row in rows) if item is not None]
|
|
2405
|
+
|
|
2406
|
+
|
|
2407
|
+
def _keys(value: Any) -> list[str]:
|
|
2408
|
+
if isinstance(value, Mapping):
|
|
2409
|
+
return [str(key) for key in value] + [name for item in value.values() for name in _keys(item)]
|
|
2410
|
+
return [name for item in value for name in _keys(item)] if isinstance(value, (list, tuple)) else []
|
|
2411
|
+
|
|
2412
|
+
|
|
2413
|
+
def emit(decision, value: Any, *, root: Path, paths: Iterable[str] = (), each: tuple[str, str] | None = None,
|
|
2414
|
+
render: Callable[[Any], str] | None = None, end: str = "\n") -> int:
|
|
2415
|
+
"""How a read surface prints memory: ``package`` the value (``each`` = (list key, path field): that list's
|
|
2416
|
+
rows one by one, ``kept``), then print it, one JSON line unless ``render`` makes the text. 0 when printed;
|
|
2417
|
+
1, with nothing on stdout and a plain line on stderr, when it is refused."""
|
|
2418
|
+
import sys
|
|
2419
|
+
if each and isinstance(value, Mapping) and each[0] in value:
|
|
2420
|
+
value = {**value, each[0]: kept(decision, value[each[0]], root=root, path=each[1])}
|
|
2421
|
+
shown = package(decision, value, root=root, paths=paths)
|
|
2422
|
+
try: # a field name memory supplies (a frontmatter map's key) is checked too; renaming one would garble it
|
|
2423
|
+
shown = shown if shown is None or redact(decision, names := _keys(shown), root=root) == names else None
|
|
2424
|
+
except RedactionError:
|
|
2425
|
+
shown = None
|
|
2426
|
+
if shown is None:
|
|
2427
|
+
print("egregore: this output names material held from you, so it is not shown", file=sys.stderr)
|
|
2428
|
+
return 1
|
|
2429
|
+
print(render(shown) if render else json.dumps(shown, ensure_ascii=False), end=end)
|
|
2430
|
+
return 0
|
|
2431
|
+
|
|
2432
|
+
|
|
2433
|
+
def emit_for(runtime, actor, value: Any, **options) -> int:
|
|
2434
|
+
"""``emit`` under the actor's read decision, in the runtime's memory."""
|
|
2435
|
+
from .contracts import Permission
|
|
2436
|
+
return emit(runtime.authorize(actor, Permission.READ), value, root=runtime.canonical_root, **options)
|