create-egregore 0.20.0 → 0.20.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (160) hide show
  1. package/assets/egregore-launcher.js +80 -15
  2. package/lib/local.js +4 -20
  3. package/lib/setup.js +21 -4
  4. package/package.json +2 -2
  5. package/runtime/codex/.codex/config.toml +3 -0
  6. package/runtime/codex/.codex/skills/archive/SKILL.md +1 -1
  7. package/runtime/codex/.codex/skills/branch/SKILL.md +1 -1
  8. package/runtime/codex/.codex/skills/checkup/SKILL.md +1 -1
  9. package/runtime/codex/.codex/skills/commit/SKILL.md +1 -1
  10. package/runtime/codex/.codex/skills/contribute/SKILL.md +1 -1
  11. package/runtime/codex/.codex/skills/delete-user/SKILL.md +1 -1
  12. package/runtime/codex/.codex/skills/ingest/SKILL.md +1 -1
  13. package/runtime/codex/.codex/skills/ingest-corpus/SKILL.md +29 -0
  14. package/runtime/codex/.codex/skills/ingest-notion/SKILL.md +1 -1
  15. package/runtime/codex/.codex/skills/ingest-user-interview/SKILL.md +1 -1
  16. package/runtime/codex/.codex/skills/issue/SKILL.md +1 -1
  17. package/runtime/codex/.codex/skills/me/SKILL.md +1 -1
  18. package/runtime/codex/.codex/skills/meeting/SKILL.md +1 -1
  19. package/runtime/codex/.codex/skills/note/SKILL.md +1 -1
  20. package/runtime/codex/.codex/skills/notion-connect/SKILL.md +1 -1
  21. package/runtime/codex/.codex/skills/onboarding/SKILL.md +1 -1
  22. package/runtime/codex/.codex/skills/pr/SKILL.md +1 -1
  23. package/runtime/codex/.codex/skills/pull/SKILL.md +1 -1
  24. package/runtime/codex/.codex/skills/push/SKILL.md +1 -1
  25. package/runtime/codex/.codex/skills/reflect/SKILL.md +1 -1
  26. package/runtime/codex/.codex/skills/review-pr/SKILL.md +1 -1
  27. package/runtime/codex/.codex/skills/save/SKILL.md +3 -1
  28. package/runtime/codex/.codex/skills/slack-connect/SKILL.md +1 -1
  29. package/runtime/codex/.codex/skills/summon/SKILL.md +1 -1
  30. package/runtime/codex/.codex/skills/sync-repos/SKILL.md +1 -1
  31. package/runtime/codex/.codex/skills/teams-connect/SKILL.md +1 -1
  32. package/runtime/codex/.codex/skills/telegram-connect/SKILL.md +1 -1
  33. package/runtime/codex/.codex/skills/telemetry/SKILL.md +1 -1
  34. package/runtime/codex/.codex/skills/test/SKILL.md +1 -1
  35. package/runtime/codex/.codex/skills/tutorial/SKILL.md +1 -1
  36. package/runtime/codex/.codex/skills/update/SKILL.md +1 -1
  37. package/runtime/codex/.codex/spec-manifest.json +4 -4
  38. package/runtime/codex/AGENTS.md +52 -39
  39. package/runtime/codex/bin/agent.sh +32 -8
  40. package/runtime/codex/bin/codex-render-spec.mjs +31 -34
  41. package/runtime/codex/bin/codex-sync-skills.sh +6 -1
  42. package/runtime/codex/bin/contribute-guard.sh +79 -0
  43. package/runtime/codex/bin/handoff-pr-backfill.sh +1 -1
  44. package/runtime/codex/bin/handoff-run.sh +8 -3
  45. package/runtime/codex/bin/handoff-save-egregore.sh +20 -8
  46. package/runtime/codex/bin/ingest.py +260 -5
  47. package/runtime/codex/bin/lib/git-message.sh +37 -0
  48. package/runtime/codex/bin/lib/git-sync.sh +26 -2
  49. package/runtime/codex/bin/lib/greeting.sh +2 -2
  50. package/runtime/codex/bin/search.sh +46 -4
  51. package/runtime/codex/bin/session-autosave.sh +13 -2
  52. package/runtime/codex/bin/settings.sh +63 -4
  53. package/runtime/codex/bin/worktree-create.sh +4 -1
  54. package/runtime/codex/manifest.json +5 -1
  55. package/runtime/pi/.codex/config.toml +3 -0
  56. package/runtime/pi/.codex/skills/archive/SKILL.md +1 -1
  57. package/runtime/pi/.codex/skills/branch/SKILL.md +1 -1
  58. package/runtime/pi/.codex/skills/checkup/SKILL.md +1 -1
  59. package/runtime/pi/.codex/skills/commit/SKILL.md +1 -1
  60. package/runtime/pi/.codex/skills/contribute/SKILL.md +1 -1
  61. package/runtime/pi/.codex/skills/delete-user/SKILL.md +1 -1
  62. package/runtime/pi/.codex/skills/ingest/SKILL.md +1 -1
  63. package/runtime/pi/.codex/skills/ingest-corpus/SKILL.md +29 -0
  64. package/runtime/pi/.codex/skills/ingest-notion/SKILL.md +1 -1
  65. package/runtime/pi/.codex/skills/ingest-user-interview/SKILL.md +1 -1
  66. package/runtime/pi/.codex/skills/issue/SKILL.md +1 -1
  67. package/runtime/pi/.codex/skills/me/SKILL.md +1 -1
  68. package/runtime/pi/.codex/skills/meeting/SKILL.md +1 -1
  69. package/runtime/pi/.codex/skills/note/SKILL.md +1 -1
  70. package/runtime/pi/.codex/skills/notion-connect/SKILL.md +1 -1
  71. package/runtime/pi/.codex/skills/onboarding/SKILL.md +1 -1
  72. package/runtime/pi/.codex/skills/pr/SKILL.md +1 -1
  73. package/runtime/pi/.codex/skills/pull/SKILL.md +1 -1
  74. package/runtime/pi/.codex/skills/push/SKILL.md +1 -1
  75. package/runtime/pi/.codex/skills/reflect/SKILL.md +1 -1
  76. package/runtime/pi/.codex/skills/review-pr/SKILL.md +1 -1
  77. package/runtime/pi/.codex/skills/save/SKILL.md +3 -1
  78. package/runtime/pi/.codex/skills/slack-connect/SKILL.md +1 -1
  79. package/runtime/pi/.codex/skills/summon/SKILL.md +1 -1
  80. package/runtime/pi/.codex/skills/sync-repos/SKILL.md +1 -1
  81. package/runtime/pi/.codex/skills/teams-connect/SKILL.md +1 -1
  82. package/runtime/pi/.codex/skills/telegram-connect/SKILL.md +1 -1
  83. package/runtime/pi/.codex/skills/telemetry/SKILL.md +1 -1
  84. package/runtime/pi/.codex/skills/test/SKILL.md +1 -1
  85. package/runtime/pi/.codex/skills/tutorial/SKILL.md +1 -1
  86. package/runtime/pi/.codex/skills/update/SKILL.md +1 -1
  87. package/runtime/pi/.codex/spec-manifest.json +4 -4
  88. package/runtime/pi/.pi/APPEND_SYSTEM.md +76 -39
  89. package/runtime/pi/.pi/spec-manifest.json +2 -2
  90. package/runtime/pi/AGENTS.md +52 -39
  91. package/runtime/pi/bin/agent.sh +32 -8
  92. package/runtime/pi/bin/codex-render-spec.mjs +31 -34
  93. package/runtime/pi/bin/codex-sync-skills.sh +6 -1
  94. package/runtime/pi/bin/contribute-guard.sh +79 -0
  95. package/runtime/pi/bin/handoff-pr-backfill.sh +1 -1
  96. package/runtime/pi/bin/handoff-run.sh +8 -3
  97. package/runtime/pi/bin/handoff-save-egregore.sh +20 -8
  98. package/runtime/pi/bin/ingest.py +260 -5
  99. package/runtime/pi/bin/lib/git-message.sh +37 -0
  100. package/runtime/pi/bin/lib/git-sync.sh +26 -2
  101. package/runtime/pi/bin/lib/greeting.sh +2 -2
  102. package/runtime/pi/bin/pi-render-spec.mjs +52 -26
  103. package/runtime/pi/bin/search.sh +46 -4
  104. package/runtime/pi/bin/session-autosave.sh +13 -2
  105. package/runtime/pi/bin/settings.sh +63 -4
  106. package/runtime/pi/bin/worktree-create.sh +4 -1
  107. package/runtime/pi/manifest.json +5 -1
  108. package/runtime/prime/.codex/config.toml +3 -0
  109. package/runtime/prime/.codex/skills/archive/SKILL.md +1 -1
  110. package/runtime/prime/.codex/skills/branch/SKILL.md +1 -1
  111. package/runtime/prime/.codex/skills/checkup/SKILL.md +1 -1
  112. package/runtime/prime/.codex/skills/commit/SKILL.md +1 -1
  113. package/runtime/prime/.codex/skills/contribute/SKILL.md +1 -1
  114. package/runtime/prime/.codex/skills/delete-user/SKILL.md +1 -1
  115. package/runtime/prime/.codex/skills/ingest/SKILL.md +1 -1
  116. package/runtime/prime/.codex/skills/ingest-corpus/SKILL.md +29 -0
  117. package/runtime/prime/.codex/skills/ingest-notion/SKILL.md +1 -1
  118. package/runtime/prime/.codex/skills/ingest-user-interview/SKILL.md +1 -1
  119. package/runtime/prime/.codex/skills/issue/SKILL.md +1 -1
  120. package/runtime/prime/.codex/skills/me/SKILL.md +1 -1
  121. package/runtime/prime/.codex/skills/meeting/SKILL.md +1 -1
  122. package/runtime/prime/.codex/skills/note/SKILL.md +1 -1
  123. package/runtime/prime/.codex/skills/notion-connect/SKILL.md +1 -1
  124. package/runtime/prime/.codex/skills/onboarding/SKILL.md +1 -1
  125. package/runtime/prime/.codex/skills/pr/SKILL.md +1 -1
  126. package/runtime/prime/.codex/skills/pull/SKILL.md +1 -1
  127. package/runtime/prime/.codex/skills/push/SKILL.md +1 -1
  128. package/runtime/prime/.codex/skills/reflect/SKILL.md +1 -1
  129. package/runtime/prime/.codex/skills/review-pr/SKILL.md +1 -1
  130. package/runtime/prime/.codex/skills/save/SKILL.md +3 -1
  131. package/runtime/prime/.codex/skills/slack-connect/SKILL.md +1 -1
  132. package/runtime/prime/.codex/skills/summon/SKILL.md +1 -1
  133. package/runtime/prime/.codex/skills/sync-repos/SKILL.md +1 -1
  134. package/runtime/prime/.codex/skills/teams-connect/SKILL.md +1 -1
  135. package/runtime/prime/.codex/skills/telegram-connect/SKILL.md +1 -1
  136. package/runtime/prime/.codex/skills/telemetry/SKILL.md +1 -1
  137. package/runtime/prime/.codex/skills/test/SKILL.md +1 -1
  138. package/runtime/prime/.codex/skills/tutorial/SKILL.md +1 -1
  139. package/runtime/prime/.codex/skills/update/SKILL.md +1 -1
  140. package/runtime/prime/.codex/spec-manifest.json +4 -4
  141. package/runtime/prime/.prime/agent/APPEND_SYSTEM.md +77 -40
  142. package/runtime/prime/.prime/agent/spec-manifest.json +2 -2
  143. package/runtime/prime/AGENTS.md +52 -39
  144. package/runtime/prime/bin/agent.sh +32 -8
  145. package/runtime/prime/bin/codex-render-spec.mjs +31 -34
  146. package/runtime/prime/bin/codex-sync-skills.sh +6 -1
  147. package/runtime/prime/bin/contribute-guard.sh +79 -0
  148. package/runtime/prime/bin/handoff-pr-backfill.sh +1 -1
  149. package/runtime/prime/bin/handoff-run.sh +8 -3
  150. package/runtime/prime/bin/handoff-save-egregore.sh +20 -8
  151. package/runtime/prime/bin/ingest.py +260 -5
  152. package/runtime/prime/bin/lib/git-message.sh +37 -0
  153. package/runtime/prime/bin/lib/git-sync.sh +26 -2
  154. package/runtime/prime/bin/lib/greeting.sh +2 -2
  155. package/runtime/prime/bin/prime-render-spec.mjs +52 -28
  156. package/runtime/prime/bin/search.sh +46 -4
  157. package/runtime/prime/bin/session-autosave.sh +13 -2
  158. package/runtime/prime/bin/settings.sh +63 -4
  159. package/runtime/prime/bin/worktree-create.sh +4 -1
  160. package/runtime/prime/manifest.json +5 -1
@@ -0,0 +1,79 @@
1
+ #!/usr/bin/env bash
2
+ set -u
3
+
4
+ SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
5
+ CONFIG="$SCRIPT_DIR/egregore.json"
6
+ EXPECTED_REPO=""
7
+
8
+ usage() {
9
+ echo "usage: bash bin/contribute-guard.sh [--config <path>] [--expect <owner/repo>]" >&2
10
+ exit 2
11
+ }
12
+
13
+ fail() {
14
+ echo "contribute: $1" >&2
15
+ exit "${2:-2}"
16
+ }
17
+
18
+ while [ $# -gt 0 ]; do
19
+ case "$1" in
20
+ --config)
21
+ [ $# -ge 2 ] || usage
22
+ CONFIG="$2"
23
+ shift 2
24
+ ;;
25
+ --expect)
26
+ [ $# -ge 2 ] || usage
27
+ EXPECTED_REPO="$2"
28
+ shift 2
29
+ ;;
30
+ --help|-h)
31
+ usage
32
+ ;;
33
+ *)
34
+ usage
35
+ ;;
36
+ esac
37
+ done
38
+
39
+ [ -f "$CONFIG" ] || fail "cannot read configuration at $CONFIG; refusing to select an upstream target"
40
+
41
+ UPSTREAM_URL=$(jq -er '
42
+ if has("upstream_url") then
43
+ if .upstream_url == null then ""
44
+ elif (.upstream_url | type) == "string" then .upstream_url
45
+ else error("upstream_url must be a string")
46
+ end
47
+ else ""
48
+ end
49
+ ' "$CONFIG" 2>/dev/null) ||
50
+ fail "configuration is invalid; refusing to select an upstream target"
51
+
52
+ if [ "$UPSTREAM_URL" = "none" ]; then
53
+ fail 'disabled because upstream_url is "none"; this checkout is a framework source, so use the save workflow for its configured integration branch' 3
54
+ fi
55
+
56
+ [ -n "$UPSTREAM_URL" ] || UPSTREAM_URL="https://github.com/egregore-labs/egregore.git"
57
+
58
+ case "$UPSTREAM_URL" in
59
+ https://github.com/*/*)
60
+ UPSTREAM_REPO="${UPSTREAM_URL#*github.com/}"
61
+ ;;
62
+ git@github.com:*/*)
63
+ UPSTREAM_REPO="${UPSTREAM_URL#git@github.com:}"
64
+ ;;
65
+ *)
66
+ fail "unsupported upstream_url '$UPSTREAM_URL'; expected a GitHub repository URL"
67
+ ;;
68
+ esac
69
+
70
+ UPSTREAM_REPO="${UPSTREAM_REPO%/}"
71
+ UPSTREAM_REPO="${UPSTREAM_REPO%.git}"
72
+ printf '%s\n' "$UPSTREAM_REPO" | grep -Eq '^[A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+$' ||
73
+ fail "invalid upstream repository '$UPSTREAM_REPO'; refusing external Git operations"
74
+
75
+ if [ -n "$EXPECTED_REPO" ] && [ "$EXPECTED_REPO" != "$UPSTREAM_REPO" ]; then
76
+ fail "target changed from '$EXPECTED_REPO' to '$UPSTREAM_REPO'; refusing external Git operations"
77
+ fi
78
+
79
+ printf '%s\n' "$UPSTREAM_REPO"
@@ -77,7 +77,7 @@ if [ -d "$MEMORY_DIR" ]; then
77
77
  (
78
78
  cd "$MEMORY_DIR" || exit 0
79
79
  git add -A >/dev/null 2>&1
80
- git commit -m "Backfill PR numbers for handoff: $TOPIC" >/dev/null 2>&1 || exit 0
80
+ git commit -m "chore(handoff): backfill PR numbers for $TOPIC" >/dev/null 2>&1 || exit 0
81
81
  git pull --rebase origin main --quiet >/dev/null 2>&1
82
82
  git push origin main --quiet >/dev/null 2>&1
83
83
  ) || true
@@ -36,6 +36,11 @@ set -euo pipefail
36
36
  # 1 on unrecoverable failure (file write, memory push, bad args).
37
37
 
38
38
  SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
39
+ # shellcheck source=bin/lib/git-message.sh
40
+ . "$SCRIPT_DIR/bin/lib/git-message.sh" 2>/dev/null || true
41
+ type egregore_commit >/dev/null 2>&1 || egregore_commit() {
42
+ local gd="$1" m="$3"; shift 3; git -C "$gd" commit -m "$m" "$@"
43
+ }
39
44
  CONFIG="$SCRIPT_DIR/egregore.json"
40
45
 
41
46
  if [[ "${1:-}" == "--help" || "${1:-}" == "-h" ]]; then
@@ -326,12 +331,12 @@ PID_ARTIFACTS=$!
326
331
  git add "$REL_FILE" handoffs/index.md >/dev/null 2>&1
327
332
 
328
333
  if [ -n "$RECIPIENT" ]; then
329
- MSG_COMMIT="Handoff: ${TOPIC} (to ${RECIPIENT})"
334
+ MSG_COMMIT="chore(handoff): record ${TOPIC} (to ${RECIPIENT})"
330
335
  else
331
- MSG_COMMIT="Handoff: ${TOPIC}"
336
+ MSG_COMMIT="chore(handoff): record ${TOPIC}"
332
337
  fi
333
338
 
334
- git commit -m "$MSG_COMMIT" >/dev/null 2>&1 || true
339
+ egregore_commit . "$SCRIPT_DIR" "$MSG_COMMIT" >/dev/null 2>&1 || true
335
340
 
336
341
  STASHED=0
337
342
  if [ -n "$(git status --porcelain 2>/dev/null)" ]; then
@@ -47,19 +47,16 @@ done
47
47
 
48
48
  case "$KIND" in
49
49
  autosave)
50
- MSG_PREFIX="Auto-save"
50
+ MSG_PREFIX="chore(autosave): save"
51
51
  PR_WHAT="Save session content captured by the opted-in autosave workflow."
52
52
  PR_WHY="This preserves unattended non-coding work without requiring an explicit /save."
53
53
  ;;
54
54
  *)
55
- MSG_PREFIX="Handoff"
55
+ MSG_PREFIX="chore(handoff): record"
56
56
  PR_WHAT="Save core-repository session work captured when the handoff was created."
57
57
  PR_WHY="The handoff records shared context immediately; this companion PR keeps the originating checkout's work from remaining only local."
58
58
  ;;
59
59
  esac
60
- PR_BODY=$(printf '## What\n\n- %s\n\n## Why\n\n%s Topic: `%s`.\n\n## Verification\n\nNot verified — background %s save; review code or configuration changes before merging.\n\n🤖 Saved via `bin/handoff-save-egregore.sh`\n' \
61
- "$PR_WHAT" "$PR_WHY" "$TOPIC" "$KIND")
62
-
63
60
  [ -n "$AUTHOR" ] || { echo "handoff-save-egregore: missing author" >&2; exit 1; }
64
61
 
65
62
  SCRIPT_DIR="$(cd "$(dirname "$0")/.." && pwd)"
@@ -88,6 +85,21 @@ BASE_REF="origin/$BASE_BRANCH"
88
85
  . "$SCRIPT_DIR/bin/lib/git-safe.sh" 2>/dev/null || true
89
86
  type git_add_guarded >/dev/null 2>&1 || git_add_guarded() { git add -A 2>/dev/null || true; }
90
87
 
88
+ # Shared commit/PR message mechanics (trailers + skeleton body shape).
89
+ # shellcheck source=bin/lib/git-message.sh
90
+ . "$SCRIPT_DIR/bin/lib/git-message.sh" 2>/dev/null || true
91
+ type egregore_commit >/dev/null 2>&1 || egregore_commit() {
92
+ local gd="$1" m="$3"; shift 3; git -C "$gd" commit -m "$m" "$@"
93
+ }
94
+ type egregore_pr_skeleton >/dev/null 2>&1 || egregore_pr_skeleton() {
95
+ printf '## What\n%s\n\n## Why\n%s\n\n## Verification\n%s\n\n%s\n' "$1" "$2" "$3" "$4"
96
+ }
97
+ PR_BODY=$(egregore_pr_skeleton \
98
+ "- $PR_WHAT" \
99
+ "$PR_WHY Topic: \`$TOPIC\`." \
100
+ "Not verified — background $KIND save; review code or configuration changes before merging." \
101
+ "🤖 Saved via \`bin/handoff-save-egregore.sh\`")
102
+
91
103
  # Shared non-coding classifier (auto-merge policy lives in one place).
92
104
  # shellcheck source=bin/lib/noncode.sh
93
105
  . "$SCRIPT_DIR/bin/lib/noncode.sh" 2>/dev/null || true
@@ -123,7 +135,7 @@ if [ -z "$CURRENT_BRANCH" ] || [ "$CURRENT_BRANCH" = "$BASE_BRANCH" ] \
123
135
  # hash from the autosave sweep) — same-name pushes wedge on non-FF.
124
136
  NEW_BRANCH="dev/${AUTHOR}/${KIND}-${SLUG_DATE}${BRANCH_SUFFIX:+-$BRANCH_SUFFIX}"
125
137
  git fetch origin "$BASE_BRANCH" --quiet 2>/dev/null || true
126
- git checkout -b "$NEW_BRANCH" "$BASE_REF" --quiet 2>/dev/null || \
138
+ git checkout --no-track -b "$NEW_BRANCH" "$BASE_REF" --quiet 2>/dev/null || \
127
139
  git checkout "$NEW_BRANCH" --quiet 2>/dev/null || exit 0
128
140
  CURRENT_BRANCH="$NEW_BRANCH"
129
141
  fi
@@ -131,7 +143,7 @@ fi
131
143
  # --- Commit whatever is uncommitted --------------------------------------
132
144
  if [ -n "$(git status --porcelain 2>/dev/null)" ]; then
133
145
  git_add_guarded
134
- git commit -m "${MSG_PREFIX}: ${TOPIC}" --quiet 2>/dev/null || true
146
+ egregore_commit . "$REPO_DIR" "${MSG_PREFIX} ${TOPIC}" --quiet 2>/dev/null || true
135
147
  fi
136
148
 
137
149
  # --- Rebase onto the configured base, fall back to merge on conflicts ----
@@ -182,7 +194,7 @@ else
182
194
  PR_URL=$(gh pr create \
183
195
  --base "$BASE_BRANCH" \
184
196
  --head "$CURRENT_BRANCH" \
185
- --title "${MSG_PREFIX}: ${TOPIC}" \
197
+ --title "${MSG_PREFIX} ${TOPIC}" \
186
198
  --body "$PR_BODY" \
187
199
  2>/dev/null || echo "")
188
200
  PR_NUMBER=$(echo "$PR_URL" | grep -oE '[0-9]+$' || echo "")
@@ -154,6 +154,36 @@ def chunks(text: str, limit: int = 7000) -> list[str]:
154
154
  return result
155
155
 
156
156
 
157
+ # Catalogue columns that carry provenance, by what they mean rather than by
158
+ # what any one organisation calls them. A profile's `map` already renames its
159
+ # own columns to these, so no further configuration is needed.
160
+ CATALOGUE_URL_FIELDS = ("url", "source_url", "link")
161
+ CATALOGUE_INSTITUTION_FIELDS = ("institution", "publisher", "source_type", "source")
162
+
163
+
164
+ def catalogue_provenance(catalogued: dict) -> tuple:
165
+ """The URL and institution a catalogue records for a document.
166
+
167
+ The resolver has always accepted these and ranks both above a filename,
168
+ because a filename is what someone happened to save a file as. Ingest
169
+ passed only the filename, so the stronger evidence sitting in the row it
170
+ had just read went unused.
171
+
172
+ Measured on a real archive that was not a marginal loss. Seventy articles
173
+ downloaded from two journal indexes were saved under bare article titles,
174
+ carrying no publisher token to match. The profile had rules for both
175
+ indexes and the catalogue named the index for every one of them, and still
176
+ nothing matched — the one input that could have identified them was the one
177
+ input never passed.
178
+ """
179
+ if not catalogued:
180
+ return "", ""
181
+ url = next((str(catalogued[f]) for f in CATALOGUE_URL_FIELDS if catalogued.get(f)), "")
182
+ institution = next(
183
+ (str(catalogued[f]) for f in CATALOGUE_INSTITUTION_FIELDS if catalogued.get(f)), "")
184
+ return url, institution
185
+
186
+
157
187
  def find_publisher_profile() -> Path | None:
158
188
  """Where this instance's publisher profile is, if it has one.
159
189
 
@@ -267,6 +297,17 @@ def add_path(args: argparse.Namespace) -> dict:
267
297
  )
268
298
  repair_map = text_repair.load_mapping(profile or {})
269
299
  ingested = unchanged = metadata_updated = skipped = pruned = 0
300
+ # How often progress is written. The manifest used to be saved once, after
301
+ # every file, so an interrupted run lost everything it had extracted — on a
302
+ # 921-document folder that was two and a half hours of OCR with nothing to
303
+ # show. Documents already on disk were re-extracted because nothing recorded
304
+ # them. Writing periodically means an interruption costs at most this many.
305
+ checkpoint_every = int(os.environ.get("EGREGORE_INGEST_CHECKPOINT", "25") or 0)
306
+ since_checkpoint = 0
307
+ # Read before the loop rather than after it, so a checkpoint writes the real
308
+ # deletion records instead of dropping them. Pruning still happens later; a
309
+ # mid-run save simply carries forward what the previous run recorded.
310
+ tombstones = existing_manifest.get("tombstones", [])
270
311
  errors: list[dict] = []
271
312
  seen_paths: set[str] = set()
272
313
 
@@ -390,7 +431,10 @@ def add_path(args: argparse.Namespace) -> dict:
390
431
  provenance["confidence"] = "high"
391
432
 
392
433
  if profile:
393
- resolved = source_profile.resolve(profile, filename=relative)
434
+ catalogue_url, catalogue_institution = catalogue_provenance(catalogued)
435
+ resolved = source_profile.resolve(
436
+ profile, url=catalogue_url, institution=catalogue_institution,
437
+ filename=relative)
394
438
  if resolved.publisher_class or resolved.notes:
395
439
  inferred = resolved.as_dict()
396
440
  # Keep the inference visible even when the catalogue answered,
@@ -438,12 +482,18 @@ def add_path(args: argparse.Namespace) -> dict:
438
482
  "status": "unverified",
439
483
  "ingested_at": now,
440
484
  }
485
+ since_checkpoint += 1
486
+ if checkpoint_every and since_checkpoint >= checkpoint_every:
487
+ # A crash between the document write and this one costs re-extraction
488
+ # of at most `checkpoint_every` files, never the whole run.
489
+ write_manifest(directory, source_id, org, args, boundaries, base, now,
490
+ previous, tombstones, errors)
491
+ since_checkpoint = 0
441
492
  if existing and existing.get("content_hash") == content_hash:
442
493
  metadata_updated += 1
443
494
  else:
444
495
  ingested += 1
445
496
 
446
- tombstones = existing_manifest.get("tombstones", [])
447
497
  if args.prune:
448
498
  if not input_path.is_dir():
449
499
  raise ValueError("--prune requires a directory snapshot")
@@ -465,11 +515,23 @@ def add_path(args: argparse.Namespace) -> dict:
465
515
  # Current documents always win over historical deletion records.
466
516
  tombstones = [row for row in tombstones if row["id"] not in previous]
467
517
 
518
+ write_manifest(directory, source_id, org, args, boundaries, base, now,
519
+ previous, tombstones, errors)
520
+ return {"org": org, "source": source_id, "ingested": ingested, "unchanged": unchanged, "metadata_updated": metadata_updated, "skipped": skipped, "pruned": pruned, "documents": len(previous), "metadata_root": str(directory), "data_root": str(docs_dir)}
521
+
522
+
523
+ def write_manifest(directory, source_id, org, args, boundaries, base, now,
524
+ previous, tombstones, errors) -> None:
525
+ """Persist progress. Called periodically during a run and once at the end.
526
+
527
+ One writer for both so a checkpoint and a completed run cannot record the
528
+ corpus differently.
529
+ """
468
530
  source_record = {
469
531
  "id": source_id,
470
532
  "org": org,
471
- "name": args.name or source_id,
472
- "kind": args.kind,
533
+ "name": getattr(args, "name", None) or source_id,
534
+ "kind": getattr(args, "kind", None),
473
535
  "boundaries": boundaries,
474
536
  "root_hint": base.name,
475
537
  "updated_at": now,
@@ -486,7 +548,6 @@ def add_path(args: argparse.Namespace) -> dict:
486
548
  "errors": errors,
487
549
  },
488
550
  )
489
- return {"org": org, "source": source_id, "ingested": ingested, "unchanged": unchanged, "metadata_updated": metadata_updated, "skipped": skipped, "pruned": pruned, "documents": len(previous), "metadata_root": str(directory), "data_root": str(docs_dir)}
490
551
 
491
552
 
492
553
  def cmd_add(args: argparse.Namespace) -> int:
@@ -494,6 +555,191 @@ def cmd_add(args: argparse.Namespace) -> int:
494
555
  return 0
495
556
 
496
557
 
558
+ def rewrite_frontmatter(path: Path, updates: dict) -> bool:
559
+ """Replace named frontmatter fields in place, leaving the text untouched.
560
+
561
+ A document's placement lives in two files — the manifest record and this
562
+ document's own frontmatter — and a reader may consult either. Updating one
563
+ and not the other produces a corpus that disagrees with itself, which is
564
+ worse than the stale value it replaced.
565
+
566
+ Only the named lines are rewritten. The body is never reparsed, so nothing
567
+ that survived extraction can be lost by a maintenance pass.
568
+ """
569
+ try:
570
+ text = path.read_text(encoding="utf-8")
571
+ except OSError:
572
+ return False
573
+ lines = text.split("\n")
574
+ if not lines or lines[0].strip() != "---":
575
+ return False
576
+ try:
577
+ close = next(i for i in range(1, len(lines)) if lines[i].strip() == "---")
578
+ except StopIteration:
579
+ return False
580
+ remaining = dict(updates)
581
+ for i in range(1, close):
582
+ key = lines[i].split(":", 1)[0].strip()
583
+ if key in remaining:
584
+ lines[i] = f"{key}: {json.dumps(remaining.pop(key), ensure_ascii=False)}"
585
+ # A field the original write omitted is appended rather than dropped, so a
586
+ # document ingested before a field existed still gains it.
587
+ for key, value in remaining.items():
588
+ lines.insert(close, f"{key}: {json.dumps(value, ensure_ascii=False)}")
589
+ close += 1
590
+ atomic_text(path, "\n".join(lines))
591
+ return True
592
+
593
+
594
+ def reresolve_source(args: argparse.Namespace) -> dict:
595
+ """Re-derive placement for documents already ingested.
596
+
597
+ Placement is written once, at ingest, and never revisited. That is right
598
+ for content — the text of a document does not change because a rule did —
599
+ but wrong for the rules themselves. Correcting a publisher profile changed
600
+ nothing about the corpus it governs: `add` skips on the content hash and
601
+ returns before it ever consults the profile, so it reported every document
602
+ unchanged while the corrected rule went unapplied. The only way to apply a
603
+ fix was to re-extract thousands of documents, redoing OCR to recompute a
604
+ field that depends on nothing but the filename.
605
+
606
+ So this re-runs the resolver alone. No file is reopened, no text is
607
+ re-extracted; the stored path is all the resolver ever needed.
608
+
609
+ Evidence precedence is the same as at ingest, and is the reason this is
610
+ safe to run at any time: a value the operator stated, or one the
611
+ organisation's own catalogue recorded, is never overwritten by an
612
+ inference. Only a placement this resolver derived — or the absence of one —
613
+ is open to revision.
614
+ """
615
+ config = load_config()
616
+ source_id = slug(args.source)
617
+ directory = source_dir(source_id)
618
+ manifest = load_manifest(directory)
619
+ records = manifest.get("documents") or []
620
+ if not records:
621
+ return {"source": source_id, "error": "no manifest, or no documents in it",
622
+ "metadata_root": str(directory)}
623
+
624
+ profile = publisher_profile()
625
+ if profile is None:
626
+ return {"source": source_id,
627
+ "error": "no publisher-profile.json found — nothing to resolve against"}
628
+ key = source_profile.boundary_key(profile)
629
+ if not key:
630
+ return {"source": source_id, "error": "profile declares no boundary key"}
631
+
632
+ docs_dir = data_root() / "sources" / source_id / "documents"
633
+ placed = rezoned = identified = unchanged = protected = missing = 0
634
+ changes: list = []
635
+
636
+ for record in records:
637
+ relative = record.get("source_path")
638
+ if not relative:
639
+ unchanged += 1
640
+ continue
641
+ boundaries = dict(record.get("boundaries") or {})
642
+ provenance = dict(record.get("provenance") or {})
643
+ catalogued = provenance.get("from_catalogue") or {}
644
+
645
+ # The same evidence the original ingest had, re-read from the record so
646
+ # a document identified only by its catalogue row resolves here too.
647
+ catalogue_url, catalogue_institution = catalogue_provenance(catalogued)
648
+ resolved = source_profile.resolve(
649
+ profile, url=catalogue_url, institution=catalogue_institution,
650
+ filename=relative)
651
+
652
+ # Placement and identification are decided separately. A document can
653
+ # become identifiable — a journal index naming its publisher — without
654
+ # that saying anything about which region it applies to, and reporting
655
+ # only the placement would call that document unchanged when the thing
656
+ # that decides how far to trust it has just been settled.
657
+ current = boundaries.get(key)
658
+ # Anything the operator stated or the catalogue recorded outranks an
659
+ # inference and is left exactly as it is.
660
+ derived_here = (provenance.get("derived") is True
661
+ and provenance.get("applied_boundary") == key)
662
+ zone_open = not ((current and not derived_here) or catalogued.get(key))
663
+ zone_changes = bool(zone_open and resolved.zone and resolved.zone != current)
664
+
665
+ # A tier the catalogue stated is likewise not ours to revise.
666
+ class_open = not catalogued.get("tier")
667
+ class_changes = bool(
668
+ class_open and resolved.publisher_class
669
+ and (resolved.publisher_class != provenance.get("publisher_class")
670
+ or resolved.tier != provenance.get("tier")))
671
+
672
+ if not zone_changes and not class_changes:
673
+ if not zone_open and not class_open:
674
+ protected += 1
675
+ elif current and not zone_open:
676
+ protected += 1
677
+ else:
678
+ unchanged += 1
679
+ continue
680
+
681
+ # The fresh resolution replaces the recorded inference whenever either
682
+ # half moved. Refreshing only on a class change once left a re-zoned
683
+ # document carrying the confidence and notes of the resolution it no
684
+ # longer had — a record that reported an inferred national zone with
685
+ # the confidence of a precise one.
686
+ updated_provenance = dict(provenance)
687
+ updated_provenance.update(resolved.as_dict())
688
+ if class_changes:
689
+ identified += 1
690
+ if catalogued:
691
+ updated_provenance["from_catalogue"] = catalogued
692
+ if zone_changes:
693
+ updated_provenance["applied_boundary"] = key
694
+ updated_provenance["derived"] = True
695
+ boundaries[key] = resolved.zone
696
+ if current:
697
+ rezoned += 1
698
+ else:
699
+ placed += 1
700
+
701
+ changes.append({"id": record["id"], "source_path": relative,
702
+ "zone": {"from": current, "to": boundaries.get(key)},
703
+ "class": {"from": provenance.get("publisher_class"),
704
+ "to": updated_provenance.get("publisher_class")},
705
+ "matched_by": resolved.matched_by})
706
+
707
+ if args.dry_run:
708
+ continue
709
+ record["boundaries"] = boundaries
710
+ record["provenance"] = updated_provenance
711
+ file_path = record.get("file_path")
712
+ target = (data_root() / file_path) if file_path else (docs_dir / f"{record['id']}.md")
713
+ if not rewrite_frontmatter(target, {"boundaries": boundaries,
714
+ "provenance": updated_provenance}):
715
+ missing += 1
716
+
717
+ if not args.dry_run and (placed or rezoned or identified):
718
+ manifest["documents"] = sorted(records, key=lambda d: d["id"])
719
+ atomic_json(directory / "manifest.json", manifest)
720
+
721
+ return {
722
+ "org": slug(config.get("slug") or "egregore"),
723
+ "source": source_id,
724
+ "dry_run": bool(args.dry_run),
725
+ "documents": len(records),
726
+ "placed": placed,
727
+ "rezoned": rezoned,
728
+ "identified": identified,
729
+ "unchanged": unchanged,
730
+ "protected": protected,
731
+ "frontmatter_not_found": missing,
732
+ "examples": changes[: max(0, args.examples)],
733
+ "metadata_root": str(directory),
734
+ }
735
+
736
+
737
+ def cmd_reresolve(args: argparse.Namespace) -> int:
738
+ result = reresolve_source(args)
739
+ print(json.dumps(result, indent=2, ensure_ascii=False))
740
+ return 1 if result.get("error") else 0
741
+
742
+
497
743
  def selection_relative_path(value: object) -> str:
498
744
  if not isinstance(value, str) or not value or "\x00" in value or "\\" in value:
499
745
  raise ValueError("selection contains an invalid relative path")
@@ -719,6 +965,15 @@ def parser() -> argparse.ArgumentParser:
719
965
  add.add_argument("--boundary", action="append", default=[])
720
966
  add.add_argument("--prune", action="store_true", help="tombstone files absent from this authoritative directory snapshot")
721
967
  add.set_defaults(func=cmd_add)
968
+ reresolve = sub.add_parser(
969
+ "reresolve",
970
+ help="re-derive placement from the current profile, without re-reading any document")
971
+ reresolve.add_argument("--source", required=True, help="source id to re-resolve")
972
+ reresolve.add_argument("--dry-run", action="store_true",
973
+ help="report what would change and write nothing")
974
+ reresolve.add_argument("--examples", type=int, default=10,
975
+ help="how many changed documents to list (default 10)")
976
+ reresolve.set_defaults(func=cmd_reresolve)
722
977
  add_selection = sub.add_parser("add-selection")
723
978
  add_selection.add_argument("manifest")
724
979
  add_selection.set_defaults(func=cmd_add_selection)
@@ -0,0 +1,37 @@
1
+ #!/usr/bin/env bash
2
+ # git-message.sh — shared commit/PR message mechanics, sourced by composers.
3
+ # Spec: .claude/context/commit-format.md + .claude/context/pr-format.md.
4
+ #
5
+ # Provides:
6
+ # egregore_session_trailer <hub_dir>
7
+ # Print "Egregore-Session: <id>" when <hub_dir>/.egregore-session-id
8
+ # exists; print nothing otherwise. Never fails.
9
+ # egregore_commit <git_dir> <hub_dir> <message> [git-commit flags…]
10
+ # git -C <git_dir> commit with the session trailer appended as the
11
+ # final paragraph (a valid git trailer block) when an id exists.
12
+ # egregore_pr_skeleton <what> <why> <verification> <footer>
13
+ # The one source of the skeleton PR body shape.
14
+
15
+ egregore_session_trailer() {
16
+ local sid
17
+ sid="$(cat "${1:-.}/.egregore-session-id" 2>/dev/null || true)"
18
+ [ -n "$sid" ] && printf 'Egregore-Session: %s' "$sid"
19
+ return 0
20
+ }
21
+
22
+ egregore_commit() {
23
+ local git_dir="$1" hub_dir="$2" msg="$3"
24
+ shift 3
25
+ local trailer
26
+ trailer="$(egregore_session_trailer "$hub_dir")"
27
+ if [ -n "$trailer" ]; then
28
+ git -C "$git_dir" commit -m "$msg" -m "$trailer" "$@"
29
+ else
30
+ git -C "$git_dir" commit -m "$msg" "$@"
31
+ fi
32
+ }
33
+
34
+ egregore_pr_skeleton() {
35
+ printf '## What\n%s\n\n## Why\n%s\n\n## Verification\n%s\n\n%s\n' \
36
+ "$1" "$2" "$3" "$4"
37
+ }
@@ -32,6 +32,22 @@ if [ -z "${BASE_BRANCH:-}" ]; then
32
32
  fi
33
33
  fi
34
34
 
35
+ # Migrate task branches created by older Egregore versions. `git branch
36
+ # <task> origin/<base>` implicitly made the task branch track the protected
37
+ # integration branch, so a bare push could target it when push.default was
38
+ # `upstream`. Task branches remain unpublished until the explicit save/push
39
+ # flow creates their same-name remote and sets that as upstream.
40
+ _CURRENT_TASK_BRANCH="$(git branch --show-current 2>/dev/null || true)"
41
+ case "$_CURRENT_TASK_BRANCH" in
42
+ dev/*|feature/*|bugfix/*)
43
+ _CURRENT_TASK_UPSTREAM="$(git rev-parse --abbrev-ref '@{upstream}' 2>/dev/null || true)"
44
+ if [ "$_CURRENT_TASK_UPSTREAM" = "origin/$BASE_BRANCH" ]; then
45
+ git branch --unset-upstream "$_CURRENT_TASK_BRANCH" >/dev/null 2>&1 || true
46
+ fi
47
+ ;;
48
+ esac
49
+ unset _CURRENT_TASK_BRANCH _CURRENT_TASK_UPSTREAM
50
+
35
51
  # --- Ensure pull.rebase is set (prevents "divergent branches" errors) ---
36
52
  git config pull.rebase true 2>/dev/null || true
37
53
 
@@ -185,7 +201,7 @@ _apply_framework_update() {
185
201
  FRAMEWORK_UPDATED_COUNT=$(printf '%s\n' "$_changed" | grep -c . || echo 0)
186
202
  FRAMEWORK_UPDATED_FILES=$(printf '%s\n' "$_changed" | awk '{print $NF}' | head -3 | tr '\n' ' ')
187
203
  git add bin/ .claude/ loom/ CLAUDE.md skills/ 2>/dev/null
188
- EGREGORE_FRAMEWORK_UPDATE=1 git commit -m "Auto-update Egregore framework" --quiet 2>/dev/null || true
204
+ EGREGORE_FRAMEWORK_UPDATE=1 git commit -m "chore(sync): update framework from upstream" --quiet 2>/dev/null || true
189
205
  FRAMEWORK_UPDATED="true"
190
206
  fi
191
207
  }
@@ -214,6 +230,14 @@ fi
214
230
  # Named setup_develop for continuity; it operates on $BASE_BRANCH, which is
215
231
  # "develop" by default and whatever egregore.json's base_branch says otherwise.
216
232
  setup_develop() {
233
+ # Detached HEAD (CI merge-ref checkouts, bisects): never create or switch
234
+ # branches here — `git checkout -b $BASE_BRANCH origin/$BASE_BRANCH` would
235
+ # silently replace the checked-out tree with the base branch. This is how
236
+ # CI jobs that invoke session-start machinery ended up testing develop
237
+ # instead of the PR under test. Syncing means updating the base-branch ref,
238
+ # not claiming HEAD; on a detached HEAD there is nothing safe to do.
239
+ [ -n "$(git branch --show-current 2>/dev/null)" ] || return 0
240
+
217
241
  # Ensure the base branch exists locally
218
242
  if ! git show-ref --verify --quiet "refs/heads/$BASE_BRANCH" 2>/dev/null; then
219
243
  if git show-ref --verify --quiet "refs/remotes/origin/$BASE_BRANCH" 2>/dev/null; then
@@ -267,7 +291,7 @@ setup_develop() {
267
291
  # Save uncommitted work so nothing is lost (guarded: never auto-commit
268
292
  # stray submodule pointers from sibling repos in the checkout)
269
293
  git_add_guarded
270
- git commit -m "Auto-save: uncommitted work from $CURRENT_BRANCH" --quiet 2>/dev/null || true
294
+ git commit -m "chore(autosave): save uncommitted work from $CURRENT_BRANCH" --quiet 2>/dev/null || true
271
295
  SAVED_BRANCH="$CURRENT_BRANCH"
272
296
  fi
273
297
 
@@ -13,7 +13,7 @@
13
13
  # - Telemetry emit
14
14
  # - First session welcome
15
15
  #
16
- # Inputs: SCRIPT_DIR, STATE_FILE, CONFIG, CTX_DIR, BRANCH, COMMITS_AHEAD,
16
+ # Inputs: SCRIPT_DIR, STATE_FILE, CONFIG, CTX_DIR, BRANCH, BASE_BRANCH, COMMITS_AHEAD,
17
17
  # AUTHOR, LOCAL_MODE, MEMORY_SYNCED, SAVED_BRANCH,
18
18
  # HEALTH_*, FRAMEWORK_VERSION, TIME_OF_DAY, EGREGORE_SESSION_ID,
19
19
  # FIRST_SESSION, DISPLAY_NAME_STATE, DASHBOARD_URL
@@ -515,5 +515,5 @@ else
515
515
  echo ""
516
516
  echo "IMPORTANT: Display the above greeting to the user exactly as-is (preserve the ASCII art formatting and ornamented status) on their first message. Then ask: What are you working on?"
517
517
  echo ""
518
- echo "BRANCH RULE: When the user responds with what they're working on, your FIRST action is to create a working branch: git fetch origin develop --quiet && git checkout -b dev/{author}/{topic-slug} origin/develop. Do this BEFORE any other work. Derive the topic slug from their description. If they ask a pure question with no work intent, skip branching."
518
+ echo "BRANCH RULE: When the user responds with what they're working on, your FIRST action is to create a working branch: git fetch origin $BASE_BRANCH --quiet && git checkout --no-track -b dev/{author}/{topic-slug} origin/$BASE_BRANCH. Do this BEFORE any other work. Derive the topic slug from their description. If they ask a pure question with no work intent, skip branching."
519
519
  fi