create-egregore 0.19.30 → 0.19.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/assets/egregore-launcher.js +2 -2
- package/package.json +1 -1
- package/runtime/codex/.codex/skills/create-skill/SKILL.md +29 -0
- package/runtime/codex/.codex/skills/slack-connect/SKILL.md +29 -0
- package/runtime/codex/.codex/spec-manifest.json +3 -3
- package/runtime/codex/AGENTS.md +5 -3
- package/runtime/codex/bin/ingest-surface.html +66 -0
- package/runtime/codex/bin/ingest.py +85 -19
- package/runtime/codex/bin/ingest.sh +49 -1
- package/runtime/codex/bin/ingest_graph.py +81 -1
- package/runtime/codex/bin/ingest_surface.py +81 -1
- package/runtime/codex/bin/notify.sh +113 -23
- package/runtime/codex/manifest.json +5 -1
- package/runtime/pi/.codex/skills/create-skill/SKILL.md +29 -0
- package/runtime/pi/.codex/skills/slack-connect/SKILL.md +29 -0
- package/runtime/pi/.codex/spec-manifest.json +3 -3
- package/runtime/pi/.pi/APPEND_SYSTEM.md +1 -1
- package/runtime/pi/.pi/spec-manifest.json +2 -2
- package/runtime/pi/AGENTS.md +5 -3
- package/runtime/pi/bin/ingest-surface.html +66 -0
- package/runtime/pi/bin/ingest.py +85 -19
- package/runtime/pi/bin/ingest.sh +49 -1
- package/runtime/pi/bin/ingest_graph.py +81 -1
- package/runtime/pi/bin/ingest_surface.py +81 -1
- package/runtime/pi/bin/notify.sh +113 -23
- package/runtime/pi/manifest.json +5 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
#!/usr/bin/env node
|
|
2
|
-
// egregore-launcher-version: 0.19.
|
|
2
|
+
// egregore-launcher-version: 0.19.32
|
|
3
3
|
//
|
|
4
4
|
// egregore — launch and browse Egregore instances from your terminal.
|
|
5
5
|
// Installed by create-egregore. Reads ~/.egregore/instances.json.
|
|
@@ -33,7 +33,7 @@ const crypto = require("node:crypto");
|
|
|
33
33
|
const { spawnSync, spawn } = require("node:child_process");
|
|
34
34
|
const readline = require("node:readline");
|
|
35
35
|
|
|
36
|
-
const VERSION = "0.19.
|
|
36
|
+
const VERSION = "0.19.32";
|
|
37
37
|
const REGISTRY = path.join(os.homedir(), ".egregore", "instances.json");
|
|
38
38
|
const DEFAULT_FILE = path.join(os.homedir(), ".egregore", "default");
|
|
39
39
|
const UPDATE_CACHE_FILE = path.join(os.homedir(), ".egregore", "update-check.json");
|
package/package.json
CHANGED
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: create-skill
|
|
3
|
+
description: 'Create a skill your org owns — scaffolded in this Egregore, protected from framework updates, shared with teammates via /save.'
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
<!-- generated-by: bin/codex-sync-skills.sh -->
|
|
7
|
+
|
|
8
|
+
# Egregore create-skill Adapter
|
|
9
|
+
|
|
10
|
+
This is fallback coverage for a long-tail Egregore workflow that has not been
|
|
11
|
+
ported to a hand-written Codex-native skill yet.
|
|
12
|
+
|
|
13
|
+
Use the project shell and filesystem directly. Do not invoke Claude Code
|
|
14
|
+
commands. Translate interactive choices to structured Codex question tooling
|
|
15
|
+
when it is available; otherwise render compact numbered choices with an
|
|
16
|
+
`Other:` option and wait for the user.
|
|
17
|
+
|
|
18
|
+
1. Read `.claude/skills/create-skill/SKILL.md` for the workflow details.
|
|
19
|
+
2. Run the referenced `bin/` scripts directly from Codex.
|
|
20
|
+
3. Treat graph and publish steps as best-effort unless that workflow explicitly
|
|
21
|
+
says they are required.
|
|
22
|
+
4. For every external notification, follow
|
|
23
|
+
`.claude/context/notification-consent.md`: plan without sending, then show
|
|
24
|
+
a separate exact Send / Edit / Cancel checkpoint. Never infer notification
|
|
25
|
+
consent from the workflow request or a batch approval.
|
|
26
|
+
5. Keep local-mode behavior filesystem-first and avoid graph or notification
|
|
27
|
+
calls when `egregore.json` declares `"mode": "local"`.
|
|
28
|
+
6. Never call the deprecated `egregore-handoff` CLI for Egregore project
|
|
29
|
+
handoffs.
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: slack-connect
|
|
3
|
+
description: 'Fallback Codex adapter for the Egregore slack-connect workflow.'
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
<!-- generated-by: bin/codex-sync-skills.sh -->
|
|
7
|
+
|
|
8
|
+
# Egregore slack-connect Adapter
|
|
9
|
+
|
|
10
|
+
This is fallback coverage for a long-tail Egregore workflow that has not been
|
|
11
|
+
ported to a hand-written Codex-native skill yet.
|
|
12
|
+
|
|
13
|
+
Use the project shell and filesystem directly. Do not invoke Claude Code
|
|
14
|
+
commands. Translate interactive choices to structured Codex question tooling
|
|
15
|
+
when it is available; otherwise render compact numbered choices with an
|
|
16
|
+
`Other:` option and wait for the user.
|
|
17
|
+
|
|
18
|
+
1. Read `.claude/skills/slack-connect/SKILL.md` for the workflow details.
|
|
19
|
+
2. Run the referenced `bin/` scripts directly from Codex.
|
|
20
|
+
3. Treat graph and publish steps as best-effort unless that workflow explicitly
|
|
21
|
+
says they are required.
|
|
22
|
+
4. For every external notification, follow
|
|
23
|
+
`.claude/context/notification-consent.md`: plan without sending, then show
|
|
24
|
+
a separate exact Send / Edit / Cancel checkpoint. Never infer notification
|
|
25
|
+
consent from the workflow request or a batch approval.
|
|
26
|
+
5. Keep local-mode behavior filesystem-first and avoid graph or notification
|
|
27
|
+
calls when `egregore.json` declares `"mode": "local"`.
|
|
28
|
+
6. Never call the deprecated `egregore-handoff` CLI for Egregore project
|
|
29
|
+
handoffs.
|
|
@@ -29,7 +29,7 @@
|
|
|
29
29
|
{
|
|
30
30
|
"section": "On Launch — MANDATORY FIRST ACTION",
|
|
31
31
|
"action": "replace",
|
|
32
|
-
"sourceHash": "
|
|
32
|
+
"sourceHash": "233d160f9ca8",
|
|
33
33
|
"note": "Claude SessionStart hook does not run on Codex; the egregore launcher renders the card via bin/codex-session-start.sh before the session. Greeting-replay instruction dropped (card is already on screen)."
|
|
34
34
|
},
|
|
35
35
|
{
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
{
|
|
42
42
|
"section": "Config Files",
|
|
43
43
|
"action": "keep",
|
|
44
|
-
"sourceHash": "
|
|
44
|
+
"sourceHash": "4ca75fe428ca",
|
|
45
45
|
"note": "Kept verbatim — shell facts, runtime-neutral."
|
|
46
46
|
},
|
|
47
47
|
{
|
|
@@ -101,7 +101,7 @@
|
|
|
101
101
|
{
|
|
102
102
|
"section": "Command Awareness",
|
|
103
103
|
"action": "keep",
|
|
104
|
-
"sourceHash": "
|
|
104
|
+
"sourceHash": "905ad739e18c",
|
|
105
105
|
"note": "Prepended the Codex command surface (skill tokens, native vs adapter split, $save abstraction); disambiguation map kept with /x -> $x token rendering. Tokens with no Codex skill (e.g. /loop, a Claude Code harness feature) are left as-is."
|
|
106
106
|
},
|
|
107
107
|
{
|
package/runtime/codex/AGENTS.md
CHANGED
|
@@ -216,7 +216,7 @@ If the startup card output contains `onboarding_needed`, invoke the `$onboarding
|
|
|
216
216
|
|
|
217
217
|
## Config Files
|
|
218
218
|
|
|
219
|
-
- **`egregore.json`** — committed. Non-secret org config: `org_name`, `github_org`, `memory_repo`, `slug`, `mode`. `api_url` is connected-mode only. Optional `boundary { posture, read[], locked }` sets the org's isolation posture (see Environment Isolation). **Never put secrets here.**
|
|
219
|
+
- **`egregore.json`** — committed. Non-secret org config: `org_name`, `github_org`, `memory_repo`, `slug`, `mode`. `api_url` is connected-mode only. Optional `boundary { posture, read[], locked }` sets the org's isolation posture (see Environment Isolation). Optional `owned_skills[]` names org-authored skills in `.claude/skills/` that `$update` must never overwrite (created via `$create-skill`). **Never put secrets here.**
|
|
220
220
|
- **`.env`** — gitignored. Personal secrets. Local mode: `GITHUB_TOKEN` only. Connected mode: `GITHUB_TOKEN` + `EGREGORE_API_KEY`. **Never use `source .env`** — use `grep '^KEY=' .env | cut -d'=' -f2-`.
|
|
221
221
|
|
|
222
222
|
In connected mode, infrastructure credentials (Neo4j, Telegram) live on the API server only — `bin/graph.sh` and `bin/notify.sh` route through the API gateway.
|
|
@@ -395,9 +395,10 @@ Invoke commands from user intent — don't wait for the slash. Each command file
|
|
|
395
395
|
**Knowledge** — `$search` `$deep-reflect` `$archive` `$note` `$add` `$meeting` `$ingest` `$scroll` `$mock` `$audit`
|
|
396
396
|
**Identity** — `$me` (view profile or set display name)
|
|
397
397
|
**Coordination** — `$ask` `$quest` `$issue` `$invite` `$delete-user` `$announce`
|
|
398
|
-
**Connectors** — `$telegram-connect` (Telegram group setup) `$teams-connect` (Microsoft Teams channel setup)
|
|
398
|
+
**Connectors** — `$notion-connect` (Notion workspace) `$telegram-connect` (Telegram group setup) `$teams-connect` (Microsoft Teams channel setup)
|
|
399
399
|
**Git** — `$branch` `$commit` `$push` `$pr` `$save` `$review-pr` `$contribute`
|
|
400
400
|
**Spirits** — `$summon` (persistent agent processes)
|
|
401
|
+
**Skills** — `$create-skill` (org-owned skill: scaffold, protect from updates, share)
|
|
401
402
|
**Infra** — `$setup` `$update` `$pull` `$env` `$infra` `$sync-repos` `$release` `$checkup`
|
|
402
403
|
|
|
403
404
|
**Disambiguation:**
|
|
@@ -410,11 +411,12 @@ Invoke commands from user intent — don't wait for the slash. Each command file
|
|
|
410
411
|
- Tasks: `$todo` (personal) · `$quest` (team exploration) · `$issue` (something broken)
|
|
411
412
|
- Questions: `$ask [person]` (async) · just ask (agent answers from context)
|
|
412
413
|
- Ingestion: `$ingest <file-or-folder>` (org-scoped corpus intake) · `$ingest meeting` · `$ingest user-interview` · `$ingest google` · ambiguous → ask which type
|
|
413
|
-
- Connectors: `$telegram-connect` (Telegram group) · `$teams-connect` (MS Teams channel) · `$ingest` (bring content in)
|
|
414
|
+
- Connectors: `$notion-connect` (Notion workspace) · `$telegram-connect` (Telegram group) · `$teams-connect` (MS Teams channel) · `$ingest` (bring content in) · `$ingest notion` (promote Notion docs)
|
|
414
415
|
- Identity: `$me` — "who am I", "call me oz"
|
|
415
416
|
- People: `$invite` (add) · `$delete-user` (remove)
|
|
416
417
|
- PRs: `$pr` (create) · `$review-pr` (review)
|
|
417
418
|
- Contributing: `$contribute` (upstream framework) · `$save` (org repo) · `$issue` (report bug)
|
|
419
|
+
- Skills: `$create-skill` (new org-owned skill) · `$contribute` (change a framework skill for everyone)
|
|
418
420
|
- Agents: `$summon` (design through questions) · `/loop` (quick recurring schedule)
|
|
419
421
|
- Announcements: `$announce` (broadcast to group) · `$handoff` (structured to a person) · `bin/notify.sh send` (DM one person)
|
|
420
422
|
|
|
@@ -143,6 +143,14 @@ __EGREGORE_DESIGN_CSS__
|
|
|
143
143
|
.button { border-radius: 99px; padding: 11px 18px; border: 1px solid var(--border); background: var(--card-bg); color: var(--dark); }
|
|
144
144
|
.button.primary { border-color: var(--terracotta); background: var(--terracotta); color: var(--paper); }
|
|
145
145
|
.button:disabled { cursor: not-allowed; color: var(--warm-gray); border-color: var(--border); background: var(--card-bg); }
|
|
146
|
+
.connectors { margin-top: 18px; padding-top: 16px; border-top: 1px solid var(--border); }
|
|
147
|
+
.connectors-head { display: flex; justify-content: space-between; align-items: baseline; }
|
|
148
|
+
.connectors-head h3 { font-size: 13px; font-weight: 600; color: var(--dark); }
|
|
149
|
+
.connector-row { display: flex; gap: 10px; margin-top: 10px; flex-wrap: wrap; }
|
|
150
|
+
.connector { display: flex; flex-direction: column; gap: 3px; text-align: left; border: 1px solid var(--border); border-radius: 12px; padding: 12px 16px; background: var(--card-bg); color: var(--dark); min-width: 180px; }
|
|
151
|
+
.connector:hover { border-color: var(--terracotta); }
|
|
152
|
+
.connector span { color: var(--muted); font-size: 12px; }
|
|
153
|
+
.connector-note { margin-top: 10px; color: var(--muted); font-size: 12px; line-height: 1.5; }
|
|
146
154
|
.status { border-left: 2px solid var(--terracotta); padding: 10px 12px; color: var(--muted); font-size: 12px; line-height: 1.5; }
|
|
147
155
|
.status.error { border-color: var(--danger); color: var(--danger); }
|
|
148
156
|
.status.done { border-color: var(--success); color: var(--success); }
|
|
@@ -198,6 +206,11 @@ __EGREGORE_DESIGN_CSS__
|
|
|
198
206
|
</div>
|
|
199
207
|
<input id="file-input" type="file" multiple>
|
|
200
208
|
<input id="folder-input" type="file" webkitdirectory multiple>
|
|
209
|
+
<div class="connectors hidden" id="connectors">
|
|
210
|
+
<div class="connectors-head"><h3>Or bring a connected source</h3><span class="step">guided setup</span></div>
|
|
211
|
+
<div class="connector-row" id="connector-row"></div>
|
|
212
|
+
<p class="connector-note">Setup continues in the terminal — the agent walks you through authorization, and your tokens stay on this machine.</p>
|
|
213
|
+
</div>
|
|
201
214
|
</section>
|
|
202
215
|
|
|
203
216
|
<section class="panel hidden" id="review-panel">
|
|
@@ -273,6 +286,59 @@ __EGREGORE_DESIGN_CSS__
|
|
|
273
286
|
document.getElementById('completion').classList.remove('hidden');
|
|
274
287
|
document.title = 'Return to the terminal — Egregore';
|
|
275
288
|
}
|
|
289
|
+
var connectorMeta = {
|
|
290
|
+
google: { label: 'Google Workspace', detail: 'Gmail, Drive, Calendar — multiple accounts' },
|
|
291
|
+
notion: { label: 'Notion', detail: 'Pages and databases' }
|
|
292
|
+
};
|
|
293
|
+
function showConnectorCompletion(provider) {
|
|
294
|
+
var meta = connectorMeta[provider] || { label: provider };
|
|
295
|
+
var account = (capabilities.connector_accounts || {})[provider] || '';
|
|
296
|
+
document.getElementById('completion-kicker').textContent = account
|
|
297
|
+
? 'Already connected'
|
|
298
|
+
: 'Guided setup continues in the terminal';
|
|
299
|
+
document.getElementById('completion-copy').textContent = account
|
|
300
|
+
? meta.label + ' is already connected as ' + account + '. Back in the terminal, your agent can add another account or start bringing material in.'
|
|
301
|
+
: 'Your agent is waiting there to connect ' + meta.label + ' — it walks you through authorization step by step. Tokens stay on this machine.';
|
|
302
|
+
document.getElementById('completion-source').textContent = account || meta.label;
|
|
303
|
+
document.getElementById('completion-files').textContent = account ? 'connected' : 'connected source';
|
|
304
|
+
document.getElementById('completion-outcome').textContent = account ? meta.label : 'guided setup';
|
|
305
|
+
document.querySelector('.shell').classList.add('hidden');
|
|
306
|
+
document.getElementById('completion').classList.remove('hidden');
|
|
307
|
+
document.title = 'Return to the terminal — Egregore';
|
|
308
|
+
}
|
|
309
|
+
function chooseConnector(provider) {
|
|
310
|
+
if (busy) return;
|
|
311
|
+
busy = true;
|
|
312
|
+
post('/api/connector', {
|
|
313
|
+
method: 'POST',
|
|
314
|
+
headers: {'Content-Type': 'application/json'},
|
|
315
|
+
body: JSON.stringify({ provider: provider })
|
|
316
|
+
}).then(function (result) {
|
|
317
|
+
if (result.redirect) {
|
|
318
|
+
// The CLI pre-started the OAuth listener — continue this same tab
|
|
319
|
+
// straight into the sign-in page.
|
|
320
|
+
window.location.href = result.redirect;
|
|
321
|
+
return;
|
|
322
|
+
}
|
|
323
|
+
showConnectorCompletion(provider);
|
|
324
|
+
}).catch(function (err) {
|
|
325
|
+
busy = false;
|
|
326
|
+
setStatus(err.message, 'error');
|
|
327
|
+
statusBox.classList.remove('hidden');
|
|
328
|
+
});
|
|
329
|
+
}
|
|
330
|
+
(function renderConnectors() {
|
|
331
|
+
var list = (capabilities.connectors || []).filter(function (p) { return connectorMeta[p]; });
|
|
332
|
+
if (!list.length) return;
|
|
333
|
+
var row = document.getElementById('connector-row');
|
|
334
|
+
row.innerHTML = list.map(function (p) {
|
|
335
|
+
return '<button class="connector" type="button" data-provider="' + p + '"><strong>' + connectorMeta[p].label + '</strong><span>' + connectorMeta[p].detail + '</span></button>';
|
|
336
|
+
}).join('');
|
|
337
|
+
row.querySelectorAll('.connector').forEach(function (btn) {
|
|
338
|
+
btn.addEventListener('click', function () { chooseConnector(btn.dataset.provider); });
|
|
339
|
+
});
|
|
340
|
+
document.getElementById('connectors').classList.remove('hidden');
|
|
341
|
+
})();
|
|
276
342
|
function choose(files) {
|
|
277
343
|
var seen = new Set();
|
|
278
344
|
selected = Array.from(files).filter(function (file) {
|
|
@@ -23,6 +23,11 @@ from datetime import datetime, timezone
|
|
|
23
23
|
from pathlib import Path
|
|
24
24
|
from xml.etree import ElementTree
|
|
25
25
|
|
|
26
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
27
|
+
import doc_segment # noqa: E402 (same-directory module, path set above)
|
|
28
|
+
import pdf_extract # noqa: E402
|
|
29
|
+
import source_profile # noqa: E402
|
|
30
|
+
|
|
26
31
|
|
|
27
32
|
ROOT = Path(__file__).resolve().parent.parent
|
|
28
33
|
CONFIG = ROOT / "egregore.json"
|
|
@@ -99,31 +104,32 @@ def parse_boundaries(items: list[str]) -> dict[str, str]:
|
|
|
99
104
|
return dict(sorted(result.items()))
|
|
100
105
|
|
|
101
106
|
|
|
102
|
-
def read_text(path: Path) -> tuple[str | None, str | None, str | None]:
|
|
107
|
+
def read_text(path: Path) -> tuple[str | None, str | None, str | None, dict | None]:
|
|
108
|
+
"""Extract text. Returns (text, error, extractor, detail).
|
|
109
|
+
|
|
110
|
+
`detail` carries per-page extraction facts for formats that have pages, so the
|
|
111
|
+
manifest records which pages needed OCR and which remain sparse or blank.
|
|
112
|
+
"""
|
|
103
113
|
ext = path.suffix.lower()
|
|
104
114
|
try:
|
|
105
115
|
if ext in TEXT_EXTS:
|
|
106
|
-
return path.read_text(encoding="utf-8", errors="replace"), None, "builtin-text"
|
|
116
|
+
return path.read_text(encoding="utf-8", errors="replace"), None, "builtin-text", None
|
|
107
117
|
if ext in HTML_EXTS:
|
|
108
118
|
raw = path.read_text(encoding="utf-8", errors="replace")
|
|
109
119
|
raw = re.sub(r"(?is)<(script|style).*?>.*?</\1>", " ", raw)
|
|
110
|
-
return html.unescape(re.sub(r"(?s)<[^>]+>", " ", raw)), None, "builtin-html"
|
|
120
|
+
return html.unescape(re.sub(r"(?s)<[^>]+>", " ", raw)), None, "builtin-html", None
|
|
111
121
|
if ext == ".docx":
|
|
112
122
|
with zipfile.ZipFile(path) as archive:
|
|
113
123
|
raw = archive.read("word/document.xml")
|
|
114
124
|
root = ElementTree.fromstring(raw)
|
|
115
|
-
|
|
125
|
+
text = "\n".join(t.text or "" for t in root.iter() if t.tag.endswith("}t"))
|
|
126
|
+
return text, None, "builtin-docx", None
|
|
116
127
|
if ext == ".pdf":
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
return None, "pdf requires a text-layer or runtime-native extraction", None
|
|
120
|
-
proc = subprocess.run([tool, str(path), "-"], check=False, capture_output=True, text=True)
|
|
121
|
-
if proc.returncode:
|
|
122
|
-
return None, f"pdftotext failed ({proc.returncode})", None
|
|
123
|
-
return proc.stdout, None, "pdftotext"
|
|
128
|
+
result = pdf_extract.extract(path)
|
|
129
|
+
return result.text, result.error, result.extractor, (result.report or None)
|
|
124
130
|
except (OSError, ValueError, zipfile.BadZipFile, ElementTree.ParseError) as exc:
|
|
125
|
-
return None, str(exc), None
|
|
126
|
-
return None, f"unsupported extension {ext or '(none)'}", None
|
|
131
|
+
return None, str(exc), None, None
|
|
132
|
+
return None, f"unsupported extension {ext or '(none)'}", None, None
|
|
127
133
|
|
|
128
134
|
|
|
129
135
|
def chunks(text: str, limit: int = 7000) -> list[str]:
|
|
@@ -146,6 +152,23 @@ def chunks(text: str, limit: int = 7000) -> list[str]:
|
|
|
146
152
|
return result
|
|
147
153
|
|
|
148
154
|
|
|
155
|
+
def publisher_profile() -> dict | None:
|
|
156
|
+
"""Load the instance's publisher profile, if one is configured.
|
|
157
|
+
|
|
158
|
+
Resolution is recorded as evidence only. It never overrides a boundary the
|
|
159
|
+
operator set: deriving a region is an inference, and an inference must not
|
|
160
|
+
silently replace a stated fact.
|
|
161
|
+
"""
|
|
162
|
+
override = os.environ.get("EGREGORE_PUBLISHER_PROFILE")
|
|
163
|
+
path = Path(override) if override else (metadata_root() / "publisher-profile.json")
|
|
164
|
+
if not path.is_file():
|
|
165
|
+
return None
|
|
166
|
+
try:
|
|
167
|
+
return json.loads(path.read_text(encoding="utf-8"))
|
|
168
|
+
except (OSError, json.JSONDecodeError):
|
|
169
|
+
return None
|
|
170
|
+
|
|
171
|
+
|
|
149
172
|
def source_dir(source_id: str) -> Path:
|
|
150
173
|
return metadata_root() / "sources" / slug(source_id)
|
|
151
174
|
|
|
@@ -200,6 +223,7 @@ def add_path(args: argparse.Namespace) -> dict:
|
|
|
200
223
|
existing_manifest = load_manifest(directory)
|
|
201
224
|
previous = {d["id"]: d for d in existing_manifest.get("documents", [])}
|
|
202
225
|
now = datetime.now(timezone.utc).isoformat()
|
|
226
|
+
profile = publisher_profile()
|
|
203
227
|
ingested = unchanged = metadata_updated = skipped = pruned = 0
|
|
204
228
|
errors: list[dict] = []
|
|
205
229
|
seen_paths: set[str] = set()
|
|
@@ -224,8 +248,10 @@ def add_path(args: argparse.Namespace) -> dict:
|
|
|
224
248
|
text, error = registered["text"], None
|
|
225
249
|
extraction = {"kind": "runtime", "extractor": registered["extractor"]}
|
|
226
250
|
else:
|
|
227
|
-
text, error, extractor = read_text(path)
|
|
251
|
+
text, error, extractor, detail = read_text(path)
|
|
228
252
|
extraction = {"kind": "local", "extractor": extractor} if extractor else None
|
|
253
|
+
if extraction and detail:
|
|
254
|
+
extraction["pages"] = detail
|
|
229
255
|
if error or text is None or not text.strip():
|
|
230
256
|
skipped += 1
|
|
231
257
|
errors.append({"path": relative, "error": error or "empty extracted text"})
|
|
@@ -255,13 +281,49 @@ def add_path(args: argparse.Namespace) -> dict:
|
|
|
255
281
|
):
|
|
256
282
|
unchanged += 1
|
|
257
283
|
continue
|
|
258
|
-
|
|
284
|
+
# Segment before chunking so no passage straddles two works. A document
|
|
285
|
+
# holding a single work yields one segment, and chunking is unchanged.
|
|
286
|
+
segments = doc_segment.segment(text)
|
|
287
|
+
document_chunks: list[tuple[str, doc_segment.Segment]] = []
|
|
288
|
+
for seg in segments:
|
|
289
|
+
for piece in chunks(seg.text(text)):
|
|
290
|
+
document_chunks.append((piece, seg))
|
|
291
|
+
|
|
259
292
|
chunk_rows = []
|
|
260
293
|
body = []
|
|
261
|
-
for index, content in enumerate(document_chunks, 1):
|
|
294
|
+
for index, (content, seg) in enumerate(document_chunks, 1):
|
|
262
295
|
chunk_id = f"{doc_id}-c{index:04d}-{digest(content, 8)}"
|
|
263
|
-
|
|
296
|
+
row = {"id": chunk_id, "index": index, "sha256": hashlib.sha256(content.encode()).hexdigest()}
|
|
297
|
+
if seg.kind != "article" or seg.label:
|
|
298
|
+
row["segment"] = {"kind": seg.kind, **({"label": seg.label} if seg.label else {})}
|
|
299
|
+
chunk_rows.append(row)
|
|
264
300
|
body.append(f"## Chunk {index:04d}\n\n<!-- ingest-chunk: {chunk_id} -->\n\n{content}")
|
|
301
|
+
|
|
302
|
+
structure = {
|
|
303
|
+
"segments": len([s for s in segments if s.kind == "article"]),
|
|
304
|
+
"summary": doc_segment.summarise(segments),
|
|
305
|
+
}
|
|
306
|
+
derived_year = doc_segment.publication_year(segments)
|
|
307
|
+
if derived_year:
|
|
308
|
+
structure["publication_year"] = derived_year
|
|
309
|
+
structure["year_source"] = "running_head"
|
|
310
|
+
|
|
311
|
+
# Fill in boundaries the operator left blank, when the instance supplies a
|
|
312
|
+
# profile that can work them out from where the document came from. People
|
|
313
|
+
# dumping a folder of documents will not type these in, and a boundary that
|
|
314
|
+
# is absent filters nothing. A derived value is marked as derived: anything
|
|
315
|
+
# downstream can tell it apart from one a person stated, and weigh it by the
|
|
316
|
+
# confidence recorded alongside. An operator's value is never overwritten.
|
|
317
|
+
provenance = None
|
|
318
|
+
doc_boundaries = dict(boundaries)
|
|
319
|
+
if profile:
|
|
320
|
+
resolved = source_profile.resolve(profile, filename=relative)
|
|
321
|
+
if resolved.publisher_class or resolved.notes:
|
|
322
|
+
provenance = resolved.as_dict()
|
|
323
|
+
key = source_profile.boundary_key(profile)
|
|
324
|
+
if key and resolved.zone and key not in doc_boundaries:
|
|
325
|
+
doc_boundaries[key] = resolved.zone
|
|
326
|
+
provenance = {**(provenance or {}), "applied_boundary": key, "derived": True}
|
|
265
327
|
out_path = docs_dir / f"{doc_id}.md"
|
|
266
328
|
frontmatter = {
|
|
267
329
|
"id": doc_id,
|
|
@@ -274,8 +336,10 @@ def add_path(args: argparse.Namespace) -> dict:
|
|
|
274
336
|
"normalized_content_hash": normalized_hash,
|
|
275
337
|
"revision": content_hash[:16],
|
|
276
338
|
"status": "unverified",
|
|
277
|
-
"boundaries":
|
|
339
|
+
"boundaries": doc_boundaries,
|
|
278
340
|
"extraction": extraction,
|
|
341
|
+
"structure": structure,
|
|
342
|
+
"provenance": provenance,
|
|
279
343
|
"ingested_at": now,
|
|
280
344
|
}
|
|
281
345
|
yaml_lines = ["---"] + [f"{key}: {json.dumps(value, ensure_ascii=False)}" for key, value in frontmatter.items()] + ["---", ""]
|
|
@@ -292,7 +356,9 @@ def add_path(args: argparse.Namespace) -> dict:
|
|
|
292
356
|
"bytes": size_bytes,
|
|
293
357
|
"chunks": chunk_rows,
|
|
294
358
|
"extraction": extraction,
|
|
295
|
-
"
|
|
359
|
+
"structure": structure,
|
|
360
|
+
"provenance": provenance,
|
|
361
|
+
"boundaries": doc_boundaries,
|
|
296
362
|
"status": "unverified",
|
|
297
363
|
"ingested_at": now,
|
|
298
364
|
}
|
|
@@ -204,8 +204,56 @@ cmd_search() {
|
|
|
204
204
|
(.[] | "\n### " + (.title // .file) + "\n" + .file + "\n\n" + (.snippet // ""))'
|
|
205
205
|
}
|
|
206
206
|
|
|
207
|
+
cmd_select() {
|
|
208
|
+
# Stitch the Google sign-in into the picker itself: when the shim ships and
|
|
209
|
+
# no account is connected yet, pre-start the OAuth listener and hand its
|
|
210
|
+
# consent URL to the surface — clicking Google then continues the SAME tab
|
|
211
|
+
# into sign-in instead of bouncing through the terminal. Already connected:
|
|
212
|
+
# pass the account so the page says so honestly. Everything degrades to the
|
|
213
|
+
# terminal-guided flow when the URL isn't ready in time.
|
|
214
|
+
local auth_urls="{}" accounts="{}" auth_pid="" auth_log="" result status=0
|
|
215
|
+
if [ -x "$ROOT/bin/connector-google.sh" ]; then
|
|
216
|
+
local active_account
|
|
217
|
+
active_account="$(jq -r '.google_account // empty' "$ROOT/.egregore-state.json" 2>/dev/null)"
|
|
218
|
+
if [ -n "$active_account" ]; then
|
|
219
|
+
accounts="$(jq -cn --arg a "$active_account" '{google: $a}')"
|
|
220
|
+
else
|
|
221
|
+
auth_log="$(mktemp)"
|
|
222
|
+
bash "$ROOT/bin/connector-google.sh" auth --print-url > "$auth_log" 2>/dev/null &
|
|
223
|
+
auth_pid=$!
|
|
224
|
+
local waited=0 url=""
|
|
225
|
+
while [ "$waited" -lt 15 ]; do
|
|
226
|
+
url="$(head -1 "$auth_log" 2>/dev/null | jq -r '.auth_url // empty' 2>/dev/null)"
|
|
227
|
+
[ -n "$url" ] && break
|
|
228
|
+
sleep 1; waited=$((waited + 1))
|
|
229
|
+
done
|
|
230
|
+
if [ -n "$url" ]; then
|
|
231
|
+
auth_urls="$(jq -cn --arg u "$url" '{google: $u}')"
|
|
232
|
+
fi
|
|
233
|
+
fi
|
|
234
|
+
fi
|
|
235
|
+
result="$(EGREGORE_CONNECTOR_AUTH_URLS="$auth_urls" EGREGORE_CONNECTOR_ACCOUNTS="$accounts" python3 "$SURFACE" "$@")" || status=$?
|
|
236
|
+
if [ -n "$auth_pid" ]; then
|
|
237
|
+
if [ "$(printf '%s' "$result" | jq -r '.auth // empty' 2>/dev/null)" = "in-browser" ]; then
|
|
238
|
+
# The user is on Google's consent page in their tab; the listener
|
|
239
|
+
# finishes the token exchange. Wait for it and fold in the outcome.
|
|
240
|
+
if wait "$auth_pid" 2>/dev/null; then
|
|
241
|
+
result="$(printf '%s' "$result" | jq -c '. + {auth: "connected"}')"
|
|
242
|
+
else
|
|
243
|
+
result="$(printf '%s' "$result" | jq -c '. + {auth: "failed"}')"
|
|
244
|
+
fi
|
|
245
|
+
else
|
|
246
|
+
kill "$auth_pid" 2>/dev/null || true
|
|
247
|
+
wait "$auth_pid" 2>/dev/null || true
|
|
248
|
+
fi
|
|
249
|
+
fi
|
|
250
|
+
[ -n "$auth_log" ] && rm -f "$auth_log"
|
|
251
|
+
printf '%s\n' "$result"
|
|
252
|
+
return "$status"
|
|
253
|
+
}
|
|
254
|
+
|
|
207
255
|
case "${1:-}" in
|
|
208
|
-
select) shift;
|
|
256
|
+
select) shift; cmd_select "$@" ;;
|
|
209
257
|
add) shift; cmd_add "$@" ;;
|
|
210
258
|
add-selection) shift; cmd_add_selection "$@" ;;
|
|
211
259
|
register-extraction) shift; python3 "$PY" register-extraction "$@" ;;
|
|
@@ -13,9 +13,10 @@ from typing import Any
|
|
|
13
13
|
|
|
14
14
|
ROOT = Path(__file__).resolve().parent.parent
|
|
15
15
|
CORPUS_SCHEMA = "egregore-ingest-corpus/v1"
|
|
16
|
+
CONNECTOR_SCHEMA = "egregore-connector-lifecycle/v1"
|
|
16
17
|
KNOWLEDGE_SCHEMA = "egregore-knowledge-projection/v1"
|
|
17
18
|
PLAN_SCHEMA = "egregore-ingest-graph-plan/v2"
|
|
18
|
-
SOURCE_KINDS = {"meeting", "interview", "google", "corpus", "document"}
|
|
19
|
+
SOURCE_KINDS = {"meeting", "interview", "google", "notion", "corpus", "document"}
|
|
19
20
|
SOURCE_RELATIONS = {"INVOLVES", "CONDUCTED_BY"}
|
|
20
21
|
ARTIFACT_RELATIONS = {"RELATES_TO", "SUPERSEDES"}
|
|
21
22
|
ARTIFACT_LABELS = {
|
|
@@ -438,6 +439,8 @@ def source_shape(kind: str) -> tuple[str, str]:
|
|
|
438
439
|
return "Interview:KnowledgeSource", "FROM_INTERVIEW"
|
|
439
440
|
if kind == "google":
|
|
440
441
|
return "Artifact:KnowledgeSource", "DERIVED_FROM"
|
|
442
|
+
if kind == "notion":
|
|
443
|
+
return "Artifact:KnowledgeSource", "DERIVED_FROM"
|
|
441
444
|
if kind == "corpus":
|
|
442
445
|
return "IngestSource:KnowledgeSource", "DERIVED_FROM"
|
|
443
446
|
return "IngestDocument:Document:KnowledgeSource", "DERIVED_FROM"
|
|
@@ -728,7 +731,84 @@ RETURN a.id""",
|
|
|
728
731
|
}
|
|
729
732
|
|
|
730
733
|
|
|
734
|
+
def connector_manifest(value: dict[str, Any]) -> bool:
|
|
735
|
+
return value.get("schema") == CONNECTOR_SCHEMA
|
|
736
|
+
|
|
737
|
+
|
|
738
|
+
def connector_plan(value: dict[str, Any], path: Path) -> dict[str, Any]:
|
|
739
|
+
"""Project connector lifecycle: which providers this org has connected,
|
|
740
|
+
which accounts are authorized on each, and which ingest sources arrived
|
|
741
|
+
through which account. Connected mode only — the plan is executed by the
|
|
742
|
+
same runner as corpus plans and is idempotent (MERGE throughout)."""
|
|
743
|
+
provider = required_string(value.get("provider"), "provider")
|
|
744
|
+
if provider not in ("google", "notion"):
|
|
745
|
+
raise ContractError(f"unsupported connector provider: {provider}")
|
|
746
|
+
accounts = value.get("accounts")
|
|
747
|
+
if not isinstance(accounts, list) or not accounts:
|
|
748
|
+
raise ContractError("accounts must be a non-empty list")
|
|
749
|
+
recorded = required_string(value.get("recorded_at"), "recorded_at")
|
|
750
|
+
source_id = str(value.get("source_id") or "")
|
|
751
|
+
queries: list[dict[str, Any]] = []
|
|
752
|
+
queries.append(
|
|
753
|
+
query(
|
|
754
|
+
"""MERGE (c:Connector {id: $provider})
|
|
755
|
+
SET c.provider = $provider,
|
|
756
|
+
c.updatedAt = datetime($recorded)
|
|
757
|
+
RETURN c.id""",
|
|
758
|
+
{"provider": provider, "recorded": recorded},
|
|
759
|
+
)
|
|
760
|
+
)
|
|
761
|
+
for entry in accounts:
|
|
762
|
+
if not isinstance(entry, dict):
|
|
763
|
+
raise ContractError("each account must be an object")
|
|
764
|
+
email = required_string(entry.get("email"), "accounts[].email")
|
|
765
|
+
status = str(entry.get("status") or "authorized")
|
|
766
|
+
if status not in ("authorized", "revoked"):
|
|
767
|
+
raise ContractError(f"unsupported account status: {status}")
|
|
768
|
+
account_id = f"{provider}:{email.lower()}"
|
|
769
|
+
queries.append(
|
|
770
|
+
query(
|
|
771
|
+
"""MERGE (a:SourceAccount {id: $id})
|
|
772
|
+
SET a.email = $email,
|
|
773
|
+
a.provider = $provider,
|
|
774
|
+
a.status = $status,
|
|
775
|
+
a.updatedAt = datetime($recorded)
|
|
776
|
+
FOREACH (_ IN CASE WHEN $status = 'authorized' AND a.authorizedAt IS NULL THEN [1] ELSE [] END |
|
|
777
|
+
SET a.authorizedAt = datetime($recorded)
|
|
778
|
+
)
|
|
779
|
+
WITH a
|
|
780
|
+
MATCH (c:Connector {id: $provider})
|
|
781
|
+
MERGE (a)-[:ON]->(c)
|
|
782
|
+
WITH a
|
|
783
|
+
OPTIONAL MATCH (s:IngestSource {id: $sourceId})
|
|
784
|
+
FOREACH (_ IN CASE WHEN s IS NULL OR $sourceId = '' THEN [] ELSE [1] END |
|
|
785
|
+
MERGE (s)-[v:VIA_ACCOUNT]->(a)
|
|
786
|
+
SET v.recordedAt = datetime($recorded)
|
|
787
|
+
)
|
|
788
|
+
RETURN a.id""",
|
|
789
|
+
{
|
|
790
|
+
"id": account_id,
|
|
791
|
+
"email": email,
|
|
792
|
+
"provider": provider,
|
|
793
|
+
"status": status,
|
|
794
|
+
"recorded": recorded,
|
|
795
|
+
"sourceId": source_id,
|
|
796
|
+
},
|
|
797
|
+
)
|
|
798
|
+
)
|
|
799
|
+
return {
|
|
800
|
+
"schema": PLAN_SCHEMA,
|
|
801
|
+
"schema_family": "connector-lifecycle",
|
|
802
|
+
"manifest_id": f"connector:{provider}",
|
|
803
|
+
"source": {"kind": "connector", "id": provider},
|
|
804
|
+
"query_count": len(queries),
|
|
805
|
+
"queries": queries,
|
|
806
|
+
}
|
|
807
|
+
|
|
808
|
+
|
|
731
809
|
def build_plan(value: dict[str, Any], path: Path) -> dict[str, Any]:
|
|
810
|
+
if connector_manifest(value):
|
|
811
|
+
return connector_plan(value, path)
|
|
732
812
|
if corpus_manifest(value):
|
|
733
813
|
return corpus_plan(value, path)
|
|
734
814
|
return knowledge_plan(value, path)
|