djournal 0.9.1 → 0.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.agents/adapters/contract.md +7 -0
- package/.agents/rules/METADATA.md +33 -0
- package/.agents/skills/decision/SKILL.md +4 -0
- package/.agents/skills/journal/SKILL.md +4 -0
- package/.agents/skills/journal-workflow/SKILL.md +16 -0
- package/.agents/skills/recall/SKILL.md +18 -0
- package/.grokbot/README.md +10 -0
- package/README.md +15 -6
- package/bin/journal.js +37 -4
- package/docs/architecture.md +14 -1
- package/docs/djournal-in-practice.md +2 -0
- package/docs/grok-bot.md +80 -0
- package/docs/installation.md +18 -3
- package/docs/read-only-team-journal.md +2 -1
- package/docs/recall.md +120 -0
- package/docs/remote-sync.md +1 -1
- package/docs/uninstalling.md +5 -2
- package/integrations/grok-bot/README.md +50 -0
- package/integrations/grok-bot/checkup.md +41 -0
- package/integrations/grok-bot/getting-started.md +50 -0
- package/integrations/grok-bot/peer-briefing.md +22 -0
- package/integrations/grok-bot/skill-registration.md +61 -0
- package/lib/installer/index.js +65 -1
- package/lib/recall/corpus.js +4 -0
- package/lib/recall/files.js +247 -0
- package/lib/recall/index.js +20 -1
- package/lib/recall/snapshot.js +8 -2
- package/package.json +4 -2
package/docs/uninstalling.md
CHANGED
|
@@ -25,16 +25,19 @@ so.
|
|
|
25
25
|
djournal uninstall --harness codex
|
|
26
26
|
djournal uninstall --harness claude-code
|
|
27
27
|
djournal uninstall --harness pi
|
|
28
|
+
djournal uninstall --harness grok-bot
|
|
28
29
|
```
|
|
29
30
|
|
|
30
31
|
## Uninstall multiple harnesses
|
|
31
32
|
|
|
32
33
|
```bash
|
|
33
|
-
djournal uninstall --harness codex,claude-code,pi
|
|
34
|
+
djournal uninstall --harness codex,claude-code,pi,grok-bot
|
|
34
35
|
```
|
|
35
36
|
|
|
36
37
|
Partial uninstall removes only the selected harness integration and keeps the
|
|
37
|
-
remaining harnesses installed.
|
|
38
|
+
remaining harnesses installed. For Grok Bot that is the owned `.grokbot/` and
|
|
39
|
+
`integrations/grok-bot/` files. Shared `.agents/` content remains until a full
|
|
40
|
+
uninstall.
|
|
38
41
|
|
|
39
42
|
## What remains durable
|
|
40
43
|
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Journal Hub recipe for Grok Bot
|
|
2
|
+
|
|
3
|
+
This directory is the shareable setup recipe for a Journal Hub bot. It is not
|
|
4
|
+
a published Team Bot or public template. Some teams cannot publish public bot
|
|
5
|
+
templates; this recipe is the open-source way to recreate the hub.
|
|
6
|
+
|
|
7
|
+
Install djournal into a project, register the portable journal skills, and
|
|
8
|
+
brief peer engineering bots. The hub does not replace those bots.
|
|
9
|
+
|
|
10
|
+
## Roles
|
|
11
|
+
|
|
12
|
+
**Journal Hub** installs and explains the journal, registers skills, and
|
|
13
|
+
reconciles shared history. Reconciliation is asynchronous: review `share`,
|
|
14
|
+
`sync`, and conflicts when asked. Do not block peer bots on that review.
|
|
15
|
+
|
|
16
|
+
**Engineering bots** write and resume their own journal entries. They use the
|
|
17
|
+
installed skills and the `djournal` / `journal` CLI against the project root.
|
|
18
|
+
They do not wait for the hub to take a turn before they can journal.
|
|
19
|
+
|
|
20
|
+
Local-only work stays unpublished until a person explicitly shares it. The hub
|
|
21
|
+
does not force publication.
|
|
22
|
+
|
|
23
|
+
## What the hub sets up
|
|
24
|
+
|
|
25
|
+
1. Confirm the project root, sync mode, peer engineering bots, and hub display
|
|
26
|
+
name by running [getting-started.md](getting-started.md). Ask one question
|
|
27
|
+
at a time.
|
|
28
|
+
2. Install the CLI (`npm install -g djournal`, or `npx djournal` for a one-off).
|
|
29
|
+
3. Run `djournal install --harness grok-bot --target <project-root>`.
|
|
30
|
+
4. Register each installed `.agents/skills/*/SKILL.md` as a Grok shared
|
|
31
|
+
workflow. Follow [skill-registration.md](skill-registration.md). Rewrite
|
|
32
|
+
paths from the install target. Do not hardcode a machine path, owner, or
|
|
33
|
+
agent id.
|
|
34
|
+
5. Register [checkup.md](checkup.md) as **Checkup with Journal Hub**.
|
|
35
|
+
6. Send [peer-briefing.md](peer-briefing.md) to the engineering bots named
|
|
36
|
+
during setup.
|
|
37
|
+
|
|
38
|
+
Grok Bot cannot emit SessionStart or Stop hooks. Do not invent hook files.
|
|
39
|
+
`AGENTS.md`, the skills, and the CLI are the harness.
|
|
40
|
+
|
|
41
|
+
## Files
|
|
42
|
+
|
|
43
|
+
| File | Use |
|
|
44
|
+
| --- | --- |
|
|
45
|
+
| `getting-started.md` | First-run questions and install steps |
|
|
46
|
+
| `skill-registration.md` | Map package skills to Grok workflow ids |
|
|
47
|
+
| `checkup.md` | Later share/sync/conflict review |
|
|
48
|
+
| `peer-briefing.md` | Short message for engineering bots |
|
|
49
|
+
|
|
50
|
+
Package docs: `docs/grok-bot.md`.
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: Checkup with Journal Hub
|
|
3
|
+
description: Review djournal status, shared work, and sync conflicts for a project without taking over engineering bots or publishing local-only work.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Checkup with Journal Hub
|
|
7
|
+
|
|
8
|
+
Use this when someone asks the hub to review journal health. Engineering bots
|
|
9
|
+
keep writing their own entries. This checkup is an asynchronous reconcile
|
|
10
|
+
pass.
|
|
11
|
+
|
|
12
|
+
## Steps
|
|
13
|
+
|
|
14
|
+
1. If `<project-root>` is not already known, ask for the directory that
|
|
15
|
+
contains `.djournal.json`. Ask only that question, then continue.
|
|
16
|
+
2. Read status and install health:
|
|
17
|
+
|
|
18
|
+
```bash
|
|
19
|
+
djournal status --target <project-root>
|
|
20
|
+
djournal doctor --target <project-root>
|
|
21
|
+
djournal work list --target <project-root>
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
3. Report which work items are `local_only` and which are shared. Do not run
|
|
25
|
+
`djournal share` unless the operator explicitly chooses a work item to
|
|
26
|
+
publish.
|
|
27
|
+
4. When sync is enabled, pull the shared projection and report conflicts:
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
djournal pull --target <project-root>
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
If the command reports conflicts, quote the conflict paths and stop. Do not
|
|
34
|
+
overwrite either side.
|
|
35
|
+
5. Run `djournal sync --target <project-root>` only for a work item the
|
|
36
|
+
operator has already shared and only when they ask to project it.
|
|
37
|
+
6. Summarize what peers should do next. Point them at the journal skills and
|
|
38
|
+
`djournal` / `journal` commands. Do not ask them to wait for another hub
|
|
39
|
+
turn before they journal.
|
|
40
|
+
|
|
41
|
+
Hooks are not part of this harness. A missing hook file is healthy.
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
---
|
|
2
|
+
name: journal-hub-getting-started
|
|
3
|
+
description: Set up djournal for Grok Bot by confirming the project root, sync mode, peer engineering bots, and hub display name, then installing the skill-only harness.
|
|
4
|
+
---
|
|
5
|
+
|
|
6
|
+
# Journal Hub getting started
|
|
7
|
+
|
|
8
|
+
Run this once when a Journal Hub bot is imported. Ask the questions below one
|
|
9
|
+
at a time. Wait for the answer before asking the next question. Use the
|
|
10
|
+
answers only for this setup. Do not invent project paths, bot names, or a
|
|
11
|
+
display name.
|
|
12
|
+
|
|
13
|
+
## Questions
|
|
14
|
+
|
|
15
|
+
1. **Project root.** Which directory should receive the journal install? It
|
|
16
|
+
must be the directory that will contain `.djournal.json` and `.agents/`.
|
|
17
|
+
Call that path `<project-root>` in later commands.
|
|
18
|
+
2. **Sync mode.** Should shared journal history live in that same repository
|
|
19
|
+
(`colocated`) or in a separate journal repository (`standalone`)? Leave sync
|
|
20
|
+
disabled until the operator asks to turn it on.
|
|
21
|
+
3. **Engineering bots.** Which peer bots should write journal entries
|
|
22
|
+
themselves? Record display names only. Agent ids are not required.
|
|
23
|
+
4. **Hub display name.** What should people call this Journal Hub?
|
|
24
|
+
|
|
25
|
+
## After the answers
|
|
26
|
+
|
|
27
|
+
1. Install the CLI if it is not already available:
|
|
28
|
+
|
|
29
|
+
```bash
|
|
30
|
+
npm install -g djournal
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
Use `npx djournal` when a global install is not appropriate.
|
|
34
|
+
|
|
35
|
+
2. Install the skill-only harness. This copies rules, skills, `AGENTS.md`, the
|
|
36
|
+
project marker, and this recipe. It does not write hook config.
|
|
37
|
+
|
|
38
|
+
```bash
|
|
39
|
+
djournal install --harness grok-bot --target <project-root>
|
|
40
|
+
```
|
|
41
|
+
|
|
42
|
+
3. Register journal skills from `<project-root>/.agents/skills` using
|
|
43
|
+
`skill-registration.md`. Resolve every rule path from `<project-root>`.
|
|
44
|
+
4. Register `checkup.md` as the shared workflow **Checkup with Journal Hub**
|
|
45
|
+
(`journal-hub-checkup`).
|
|
46
|
+
5. Send `peer-briefing.md` to each engineering bot from step 3, with
|
|
47
|
+
`<project-root>` and the hub display name filled in.
|
|
48
|
+
|
|
49
|
+
Do not publish a public bot template as part of this setup. Do not share or
|
|
50
|
+
sync work unless the operator explicitly asks.
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
# Peer engineering bot briefing
|
|
2
|
+
|
|
3
|
+
Send this to each engineering bot named during getting started. Replace the
|
|
4
|
+
angle-bracket fields. Keep the message short.
|
|
5
|
+
|
|
6
|
+
```text
|
|
7
|
+
<hub-display-name> can help with the project journal, and you can use it
|
|
8
|
+
without waiting for the hub.
|
|
9
|
+
|
|
10
|
+
Project root: <project-root>
|
|
11
|
+
Install (already done when the hub finished setup):
|
|
12
|
+
djournal install --harness grok-bot --target <project-root>
|
|
13
|
+
|
|
14
|
+
Use the journal skills and the djournal / journal CLI against that root.
|
|
15
|
+
Start meaningful work with the journal workflow skill. Resume, plan, research,
|
|
16
|
+
decide, and close entries yourself.
|
|
17
|
+
|
|
18
|
+
<hub-display-name> is an async reconciler. Ask it to run Checkup with Journal
|
|
19
|
+
Hub when you want status, share/sync review, or conflict paths. Local-only
|
|
20
|
+
work stays unpublished until someone explicitly shares it. There are no Grok
|
|
21
|
+
session hooks; skills and the CLI are the integration.
|
|
22
|
+
```
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
# Register journal skills in Grok Bot
|
|
2
|
+
|
|
3
|
+
Grok Bot loads shared workflows. It does not load `.agents/skills` by itself
|
|
4
|
+
and it has no session hooks. After `djournal install --harness grok-bot`,
|
|
5
|
+
register each installed skill with the harness **UpdateSkill** action.
|
|
6
|
+
|
|
7
|
+
Keep the upstream skill files generic. Rewrite a copy for the payload. Do not
|
|
8
|
+
edit the installed `SKILL.md` files in place, and do not embed a workspace
|
|
9
|
+
path, owner name, or agent id.
|
|
10
|
+
|
|
11
|
+
## Suggested workflow ids
|
|
12
|
+
|
|
13
|
+
| Installed skill directory | Suggested workflow id |
|
|
14
|
+
| --- | --- |
|
|
15
|
+
| `journal-workflow` | `journal-workflow` |
|
|
16
|
+
| `resume` | `journal-resume` |
|
|
17
|
+
| `init-work` | `journal-init-work` |
|
|
18
|
+
| `switch` | `journal-switch` |
|
|
19
|
+
| `journal` | `journal-entry` |
|
|
20
|
+
| `plan` | `journal-plan` |
|
|
21
|
+
| `recall` | `journal-recall` |
|
|
22
|
+
| `decision` | `journal-decision` |
|
|
23
|
+
| `document` | `journal-document` |
|
|
24
|
+
| `research-codebase` | `journal-research-codebase` |
|
|
25
|
+
| `research-web` | `journal-research-web` |
|
|
26
|
+
| `reinforce` | `journal-reinforce` |
|
|
27
|
+
| `audit` | `journal-audit` |
|
|
28
|
+
| `reconcile` | `journal-reconcile` |
|
|
29
|
+
|
|
30
|
+
Hub-only prose in this recipe is not an upstream skill:
|
|
31
|
+
|
|
32
|
+
| Recipe file | Suggested workflow id | Title |
|
|
33
|
+
| --- | --- | --- |
|
|
34
|
+
| `getting-started.md` | `journal-hub-getting-started` | Journal Hub getting started |
|
|
35
|
+
| `checkup.md` | `journal-hub-checkup` | Checkup with Journal Hub |
|
|
36
|
+
|
|
37
|
+
## Build an UpdateSkill payload
|
|
38
|
+
|
|
39
|
+
1. Set `<project-root>` to the install target (the directory that contains
|
|
40
|
+
`.djournal.json` and `.agents/`).
|
|
41
|
+
2. Read `<project-root>/.agents/skills/<directory>/SKILL.md`.
|
|
42
|
+
3. Take `name` and `description` from the YAML frontmatter. Use the suggested
|
|
43
|
+
workflow id from the table. Do not invent a project-specific id.
|
|
44
|
+
4. Rewrite the body into the UpdateSkill payload:
|
|
45
|
+
- Point rule reads at `<project-root>/.agents/rules/...`. Journal root
|
|
46
|
+
resolution stays the procedure in `<project-root>/.agents/rules/STATE.md`:
|
|
47
|
+
use `.djournal.json` in the project root when it exists, otherwise the
|
|
48
|
+
project-local `.journal`.
|
|
49
|
+
- Write CLI examples as `djournal <command> --target <project-root>` or
|
|
50
|
+
`journal <command> --target <project-root>`. Replace `<project-root>` with
|
|
51
|
+
the confirmed install target at registration time.
|
|
52
|
+
- When one skill should point at another, use a Grok sibling link:
|
|
53
|
+
`[Journal workflow](sand-workflow:journal-workflow)`. Use the suggested id
|
|
54
|
+
after `sand-workflow:`.
|
|
55
|
+
5. Register that id, name, description, and rewritten body with UpdateSkill.
|
|
56
|
+
6. Repeat for every row, then register the two hub workflows from this
|
|
57
|
+
directory the same way.
|
|
58
|
+
|
|
59
|
+
Re-run this mapping after `djournal upgrade` when skill text changes. The
|
|
60
|
+
portable package files remain the source. The registered workflows are a
|
|
61
|
+
projection for Grok Bot.
|
package/lib/installer/index.js
CHANGED
|
@@ -23,12 +23,13 @@ const {
|
|
|
23
23
|
} = require("../update-checker.js");
|
|
24
24
|
|
|
25
25
|
const SCHEMA_VERSION = 1;
|
|
26
|
-
const SUPPORTED_HARNESSES = ["codex", "claude-code", "pi"];
|
|
26
|
+
const SUPPORTED_HARNESSES = ["codex", "claude-code", "pi", "grok-bot"];
|
|
27
27
|
const FUTURE_HARNESSES = ["opencode", "zed"];
|
|
28
28
|
const HARNESS_CONFIG_PATHS = {
|
|
29
29
|
codex: ".codex/hooks.json",
|
|
30
30
|
"claude-code": ".claude/settings.json",
|
|
31
31
|
};
|
|
32
|
+
const GROK_BOT_ASSET_DIRECTORIES = [".grokbot", "integrations/grok-bot"];
|
|
32
33
|
const MANIFEST_PATH = ".journal/.install/manifest.json";
|
|
33
34
|
const TRANSACTION_PATH = ".journal/.install/transaction.json";
|
|
34
35
|
const PROJECT_MARKER_PATH = ".djournal.json";
|
|
@@ -288,6 +289,7 @@ function detectHarnesses(target, options = {}) {
|
|
|
288
289
|
"claude-code": exists("claude") || fs.existsSync(path.join(target, ".claude")) || fs.existsSync(path.join(target, "CLAUDE.md")),
|
|
289
290
|
opencode: exists("opencode") || fs.existsSync(path.join(target, ".opencode")) || fs.existsSync(path.join(target, "opencode.json")),
|
|
290
291
|
pi: exists("pi") || fs.existsSync(path.join(target, ".pi")),
|
|
292
|
+
"grok-bot": fs.existsSync(path.join(target, ".grokbot")),
|
|
291
293
|
zed: exists("zed") || fs.existsSync(path.join(target, ".zed")),
|
|
292
294
|
};
|
|
293
295
|
return {
|
|
@@ -299,6 +301,7 @@ function detectHarnesses(target, options = {}) {
|
|
|
299
301
|
|
|
300
302
|
function normalizeHarness(name) {
|
|
301
303
|
if (name === "claude") return "claude-code";
|
|
304
|
+
if (name === "grok" || name === "grokbot") return "grok-bot";
|
|
302
305
|
return name;
|
|
303
306
|
}
|
|
304
307
|
|
|
@@ -500,6 +503,16 @@ function harnessFragment(sourceRoot, harness) {
|
|
|
500
503
|
return parseJson(fs.readFileSync(path.join(sourceRoot, relative)), relative);
|
|
501
504
|
}
|
|
502
505
|
|
|
506
|
+
function listOwnedAssets(sourceRoot, directory) {
|
|
507
|
+
const absolute = path.join(sourceRoot, directory);
|
|
508
|
+
if (!fs.existsSync(absolute) || !fs.statSync(absolute).isDirectory()) {
|
|
509
|
+
throw new InstallerError(`missing harness assets: ${directory}`, "MISSING_ASSETS");
|
|
510
|
+
}
|
|
511
|
+
const files = listFiles(absolute).map((relative) => `${directory}/${relative}`);
|
|
512
|
+
if (!files.length) throw new InstallerError(`missing harness assets: ${directory}`, "MISSING_ASSETS");
|
|
513
|
+
return files;
|
|
514
|
+
}
|
|
515
|
+
|
|
503
516
|
function planInstall(sourceRoot, target, harnesses, oldManifest, store) {
|
|
504
517
|
const old = recordMap(oldManifest);
|
|
505
518
|
const records = [];
|
|
@@ -538,6 +551,14 @@ function planInstall(sourceRoot, target, harnesses, oldManifest, store) {
|
|
|
538
551
|
);
|
|
539
552
|
records.push(extension.record); operations.push(extension.op);
|
|
540
553
|
}
|
|
554
|
+
if (harnesses.includes("grok-bot")) {
|
|
555
|
+
for (const directory of GROK_BOT_ASSET_DIRECTORIES) {
|
|
556
|
+
for (const relative of listOwnedAssets(sourceRoot, directory)) {
|
|
557
|
+
const result = copyRecord(sourceRoot, target, relative, old.get(relative), "grok-bot");
|
|
558
|
+
records.push(result.record); operations.push(result.op);
|
|
559
|
+
}
|
|
560
|
+
}
|
|
561
|
+
}
|
|
541
562
|
|
|
542
563
|
const desired = new Set(records.map((record) => record.path));
|
|
543
564
|
for (const record of oldManifest?.files || []) {
|
|
@@ -1212,6 +1233,49 @@ function doctor(options) {
|
|
|
1212
1233
|
detail: `${assetStatus}; Pi project trust required`,
|
|
1213
1234
|
});
|
|
1214
1235
|
}
|
|
1236
|
+
if (loadedManifest?.harnesses?.includes("grok-bot")) {
|
|
1237
|
+
const agentsReady = fs.existsSync(path.join(target, ".agents/rules")) && fs.existsSync(path.join(target, ".agents/skills"));
|
|
1238
|
+
checks.push({
|
|
1239
|
+
name: "grok-bot agents",
|
|
1240
|
+
ok: agentsReady,
|
|
1241
|
+
detail: agentsReady ? ".agents rules and skills present" : "missing .agents rules or skills",
|
|
1242
|
+
});
|
|
1243
|
+
let markerOk = false;
|
|
1244
|
+
let markerDetail = "missing project marker";
|
|
1245
|
+
try {
|
|
1246
|
+
const marker = readProjectMarker(target);
|
|
1247
|
+
markerOk = Boolean(marker?.projectKey && marker?.journalRoot);
|
|
1248
|
+
if (markerOk) markerDetail = "project marker present";
|
|
1249
|
+
} catch (error) {
|
|
1250
|
+
markerDetail = error.message;
|
|
1251
|
+
}
|
|
1252
|
+
checks.push({ name: "grok-bot marker", ok: markerOk, detail: markerDetail });
|
|
1253
|
+
const commandAvailable = options.commandExists || commandExists;
|
|
1254
|
+
const cli = ["djournal", "journal"].find((command) => commandAvailable(command));
|
|
1255
|
+
checks.push({
|
|
1256
|
+
name: "grok-bot cli",
|
|
1257
|
+
ok: true,
|
|
1258
|
+
detail: cli
|
|
1259
|
+
? `${cli} on PATH; register skills from integrations/grok-bot/skill-registration.md; hooks are not used`
|
|
1260
|
+
: "djournal CLI not on PATH; register skills from integrations/grok-bot/skill-registration.md; hooks are not used",
|
|
1261
|
+
});
|
|
1262
|
+
const owned = loadedManifest.files.filter((item) => item.harness === "grok-bot");
|
|
1263
|
+
const statuses = owned.map((record) => recordStatus(target, record));
|
|
1264
|
+
const assetStatus = owned.length && statuses.every((item) => item === "clean" || item === "modified")
|
|
1265
|
+
? (statuses.every((item) => item === "clean") ? "clean" : "modified")
|
|
1266
|
+
: "missing";
|
|
1267
|
+
checks.push({
|
|
1268
|
+
name: "grok-bot assets",
|
|
1269
|
+
ok: assetStatus === "clean" || assetStatus === "modified",
|
|
1270
|
+
detail: `${assetStatus}; instruction/skill-only; register skills from integrations/grok-bot/skill-registration.md`,
|
|
1271
|
+
});
|
|
1272
|
+
} else if (detected.evidence["grok-bot"]) {
|
|
1273
|
+
checks.push({
|
|
1274
|
+
name: "grok-bot",
|
|
1275
|
+
ok: true,
|
|
1276
|
+
detail: "marker present; install with djournal install --harness grok-bot. Hooks are not used. See docs/grok-bot.md",
|
|
1277
|
+
});
|
|
1278
|
+
}
|
|
1215
1279
|
const { inspectSnapshot } = require("../recall/snapshot.js");
|
|
1216
1280
|
const recallCache = inspectSnapshot(projectContext(target, options), options);
|
|
1217
1281
|
checks.push({
|
package/lib/recall/corpus.js
CHANGED
|
@@ -4,6 +4,7 @@ const crypto = require("node:crypto");
|
|
|
4
4
|
const fs = require("node:fs");
|
|
5
5
|
const path = require("node:path");
|
|
6
6
|
const YAML = require("yaml");
|
|
7
|
+
const { extractReferences } = require("./files.js");
|
|
7
8
|
|
|
8
9
|
const ENTRY_DIRECTORIES = ["journal", "_research", "docs", "decisions"];
|
|
9
10
|
const MAX_MARKDOWN_BYTES = 16 * 1024 * 1024;
|
|
@@ -232,6 +233,7 @@ function documentFromDescriptor(descriptor, workContexts, warnings, existingRead
|
|
|
232
233
|
const itemId = rawId(descriptor.kind, metadata, descriptor.relative);
|
|
233
234
|
const kind = descriptor.kind;
|
|
234
235
|
const entryType = kind === "entry" ? scalar(metadata.entryType, inferredEntryType(descriptor.area, descriptor.relative)) : "work";
|
|
236
|
+
const references = extractReferences(metadata, parsed.body, warnings, descriptor.relative);
|
|
235
237
|
return {
|
|
236
238
|
document: {
|
|
237
239
|
id: `${kind}:${itemId}`,
|
|
@@ -254,6 +256,8 @@ function documentFromDescriptor(descriptor, workContexts, warnings, existingRead
|
|
|
254
256
|
updatedAt: scalar(metadata.updatedAt),
|
|
255
257
|
links: normalizeLinks(metadata.links),
|
|
256
258
|
incomingLinks: [],
|
|
259
|
+
codeReferences: references.references,
|
|
260
|
+
codeReferencesTruncated: references.truncated,
|
|
257
261
|
},
|
|
258
262
|
manifest: {
|
|
259
263
|
size: read.stat.size,
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
|
|
3
|
+
const path = require("node:path");
|
|
4
|
+
|
|
5
|
+
const EXTRACTOR_VERSION = 1;
|
|
6
|
+
const MAX_REFERENCES = 1000;
|
|
7
|
+
const MAX_RELATED_EDGES = 200;
|
|
8
|
+
const MAX_MATCHES = 100;
|
|
9
|
+
|
|
10
|
+
function usage(message) {
|
|
11
|
+
return Object.assign(new Error(message), { code: "USAGE" });
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
// Paths are lexical: deleted and not-yet-created code must remain searchable.
|
|
15
|
+
function normalizePath(value, { target, query = false } = {}) {
|
|
16
|
+
if (typeof value !== "string" || !value.trim() || value.length > 2048 || /[\x00-\x1f\x7f]/.test(value)) return null;
|
|
17
|
+
let normalized = value.trim().replace(/(?::\d+(?::\d+)?|#L\d+(?:-L?\d+)?)$/, "");
|
|
18
|
+
if (process.platform === "win32") normalized = normalized.replace(/\\/g, "/");
|
|
19
|
+
if (/^[a-z][a-z\d+.-]*:/i.test(normalized) && !path.isAbsolute(normalized)) return null;
|
|
20
|
+
if (path.isAbsolute(normalized)) {
|
|
21
|
+
if (!query || !target) return null;
|
|
22
|
+
normalized = path.relative(path.resolve(target), normalized).split(path.sep).join("/");
|
|
23
|
+
}
|
|
24
|
+
if (/[<>*?`|]/.test(normalized) || normalized.includes("${")) return null;
|
|
25
|
+
normalized = path.posix.normalize(normalized).replace(/\/$/, "");
|
|
26
|
+
if (!normalized || normalized === "." || normalized === ".." || normalized.startsWith("../") || normalized.startsWith("/")) return null;
|
|
27
|
+
if (normalized === ".journal" || normalized.startsWith(".journal/")) return null;
|
|
28
|
+
return normalized;
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
function repositoryLabel(value) {
|
|
32
|
+
return typeof value === "string" && value.trim().length > 0 && value.length <= 256 && !/[\x00-\x1f\x7f]/.test(value);
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function extractReferences(metadata, body, warnings, journalPath) {
|
|
36
|
+
const references = [];
|
|
37
|
+
const seen = new Set();
|
|
38
|
+
let truncated = false;
|
|
39
|
+
function add(raw, kind, repository, relation, provenance, bodyLine, excerpt) {
|
|
40
|
+
const normalized = normalizePath(raw);
|
|
41
|
+
if (!normalized || !["file", "directory"].includes(kind) || !["mentions", "changes", "constrains"].includes(relation)) return false;
|
|
42
|
+
if (repository !== "" && !repositoryLabel(repository)) return false;
|
|
43
|
+
const reference = { path: normalized, kind, repository, relation, provenance, bodyLine, excerpt: excerpt.slice(0, 240) };
|
|
44
|
+
const key = JSON.stringify([normalized, kind, repository, relation, provenance]);
|
|
45
|
+
if (!seen.has(key)) {
|
|
46
|
+
if (references.length >= MAX_REFERENCES) { truncated = true; return true; }
|
|
47
|
+
seen.add(key);
|
|
48
|
+
references.push(reference);
|
|
49
|
+
}
|
|
50
|
+
return true;
|
|
51
|
+
}
|
|
52
|
+
const explicit = metadata.metadata?.codeReferences;
|
|
53
|
+
if (explicit !== undefined) {
|
|
54
|
+
if (!Array.isArray(explicit)) warnings.push({ code: "INVALID_CODE_REFERENCES", path: journalPath });
|
|
55
|
+
else for (const ref of explicit) {
|
|
56
|
+
if (!ref || typeof ref !== "object" || !add(ref.path, ref.kind || "file", ref.repository ?? "", ref.relation || "mentions", "metadata", null, typeof ref.path === "string" ? ref.path : "")) {
|
|
57
|
+
if (!warnings.some(w => w.code === "INVALID_CODE_REFERENCES" && w.path === journalPath)) warnings.push({ code: "INVALID_CODE_REFERENCES", path: journalPath });
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
// Deliberately bounded syntax, not a general Markdown parser. Unsupported
|
|
62
|
+
// forms can always use codeReferences. Fenced examples never imply scope.
|
|
63
|
+
let fence = null;
|
|
64
|
+
const lines = body.split(/\r?\n/);
|
|
65
|
+
for (let i = 0; i < lines.length; i += 1) {
|
|
66
|
+
const line = lines[i];
|
|
67
|
+
const marker = line.match(/^\s{0,3}(`{3,}|~{3,})/);
|
|
68
|
+
if (marker) {
|
|
69
|
+
if (!fence) fence = marker[1];
|
|
70
|
+
else if (marker[1][0] === fence[0] && marker[1].length >= fence.length) fence = null;
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
if (fence || /^ {4}|^\t/.test(line)) continue;
|
|
74
|
+
const candidates = [];
|
|
75
|
+
for (const match of line.matchAll(/(`+)([^`\n]+)\1/g)) candidates.push([match[2], "inline_code"]);
|
|
76
|
+
for (const match of line.matchAll(/\[[^\]\n]*\]\((<[^>\n]+>|[^\s()]+)(?:\s+"[^"\n]*")?\)/g)) {
|
|
77
|
+
candidates.push([match[1].replace(/^<|>$/g, ""), "markdown_link"]);
|
|
78
|
+
}
|
|
79
|
+
if (line.trim().startsWith("|")) {
|
|
80
|
+
for (const cell of line.split("|").slice(1, -1)) if (!/[`\[\]]/.test(cell)) candidates.push([cell.trim(), "table_cell"]);
|
|
81
|
+
}
|
|
82
|
+
for (const [raw, provenance] of candidates) {
|
|
83
|
+
// Only whole path literals; commands, prose, and relative journal links
|
|
84
|
+
// must not become references just because they contain a slash.
|
|
85
|
+
if (/^(?:node|npm|npx|git|cat|rg|python\d*|bash|sh|djournal|journal)\s/.test(raw)) continue;
|
|
86
|
+
if (/\s[-]|[;&=]|^\$|^\.{2}\//.test(raw) || /^(?:https?|file):/i.test(raw)) continue;
|
|
87
|
+
const normalized = normalizePath(raw);
|
|
88
|
+
if (!normalized || raw.includes("\\") || /#(?!L\d+(?:-L?\d+)?$)/.test(raw)) continue;
|
|
89
|
+
const directory = raw.endsWith("/");
|
|
90
|
+
const basename = path.posix.basename(normalized);
|
|
91
|
+
if (!directory && !/^(?:[^\s/]+\.[a-z\d]+|\.[\w.-]+|README|LICENSE|Makefile|Dockerfile|AGENTS\.md)$/i.test(basename) && !/\.[a-z\d]+$/i.test(basename)) continue;
|
|
92
|
+
if (/\s/.test(raw) && !normalized.includes("/") && provenance === "table_cell") continue;
|
|
93
|
+
if (/^(?:journal|_research|decisions|docs)\//.test(raw) && /\d{4}-\d{2}-\d{2}-/.test(raw)) continue;
|
|
94
|
+
add(raw, directory ? "directory" : "file", "", "mentions", provenance, i + 1, line.trim());
|
|
95
|
+
}
|
|
96
|
+
}
|
|
97
|
+
if (truncated) warnings.push({ code: "CODE_REFERENCES_TRUNCATED", path: journalPath });
|
|
98
|
+
return { references, truncated };
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function postingKey(repository, file) { return JSON.stringify([repository, file]); }
|
|
102
|
+
|
|
103
|
+
function buildPathIndex(documents) {
|
|
104
|
+
const files = Object.create(null);
|
|
105
|
+
const directories = Object.create(null);
|
|
106
|
+
const repositories = new Set();
|
|
107
|
+
for (const document of Object.values(documents)) {
|
|
108
|
+
if (document.kind !== "entry") continue;
|
|
109
|
+
for (const [referenceIndex, ref] of document.codeReferences.entries()) {
|
|
110
|
+
repositories.add(ref.repository);
|
|
111
|
+
const map = ref.kind === "directory" ? directories : files;
|
|
112
|
+
const key = postingKey(ref.repository, ref.path);
|
|
113
|
+
(map[key] ||= []).push({ documentId: document.id, referenceIndex });
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
return { files, directories, repositories: [...repositories].sort() };
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function validPathIndex(snapshot) {
|
|
120
|
+
if (snapshot.extractorVersion !== EXTRACTOR_VERSION || !snapshot.pathIndex) return false;
|
|
121
|
+
for (const [key, doc] of Object.entries(snapshot.documents)) {
|
|
122
|
+
if (!doc || doc.id !== key || !Array.isArray(doc.codeReferences) || doc.codeReferences.length > MAX_REFERENCES || typeof doc.codeReferencesTruncated !== "boolean") return false;
|
|
123
|
+
for (const ref of doc.codeReferences) {
|
|
124
|
+
if (!ref || typeof ref.path !== "string" || normalizePath(ref.path) !== ref.path || !["file", "directory"].includes(ref.kind) || (ref.repository !== "" && !repositoryLabel(ref.repository)) || !["mentions", "changes", "constrains"].includes(ref.relation)) return false;
|
|
125
|
+
if (!["metadata", "inline_code", "markdown_link", "table_cell"].includes(ref.provenance) || !(ref.bodyLine === null || (Number.isInteger(ref.bodyLine) && ref.bodyLine > 0)) || typeof ref.excerpt !== "string" || ref.excerpt.length > 240) return false;
|
|
126
|
+
}
|
|
127
|
+
}
|
|
128
|
+
// Re-derive to reject dangling/forged posting references rather than trusting
|
|
129
|
+
// cached indices. This is linear in retained references, not source bodies.
|
|
130
|
+
return JSON.stringify(snapshot.pathIndex) === JSON.stringify(buildPathIndex(snapshot.documents));
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
function normalizeInputs(inputs, options = {}) {
|
|
134
|
+
if (!Array.isArray(inputs) || !inputs.length || inputs.length > 100) throw usage("recall files requires 1 to 100 paths");
|
|
135
|
+
if (options.repository !== undefined && !repositoryLabel(options.repository)) throw usage("--repository requires a non-empty label (maximum 256 characters)");
|
|
136
|
+
if (options.limit !== undefined && (!Number.isInteger(options.limit) || options.limit < 1 || options.limit > 100)) throw usage("--limit must be an integer from 1 to 100");
|
|
137
|
+
return [...new Set(inputs.map(value => {
|
|
138
|
+
const normalized = normalizePath(value, { ...options, query: true });
|
|
139
|
+
if (!normalized) throw usage("recall files requires code paths within the target checkout");
|
|
140
|
+
return normalized;
|
|
141
|
+
}))];
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function lookupPaths(snapshot, inputs, options = {}) {
|
|
145
|
+
const results = new Map();
|
|
146
|
+
const documents = snapshot.documents;
|
|
147
|
+
const index = snapshot.pathIndex;
|
|
148
|
+
const scopes = options.repository === undefined ? index.repositories : ["", options.repository];
|
|
149
|
+
const counts = new Map(inputs.map(p => [p, new Set()]));
|
|
150
|
+
const limit = options.limit || 10;
|
|
151
|
+
const warnings = [];
|
|
152
|
+
const eligible = doc => doc.kind === "entry" && (!options.work || doc.workSlug === options.work);
|
|
153
|
+
const inScope = doc => options.repository === undefined || !doc.codeReferences.length || doc.codeReferences.some(ref => !ref.repository || ref.repository === options.repository);
|
|
154
|
+
function add(doc, match) {
|
|
155
|
+
let result = results.get(doc.id);
|
|
156
|
+
if (!result) {
|
|
157
|
+
const { codeReferences, codeReferencesTruncated, ...fields } = doc;
|
|
158
|
+
result = { ...fields, id: doc.rawId, documentId: doc.id, matches: [], matchesTruncated: false, codeReferencesTruncated, incomingLinks: snapshot.incomingLinks?.[doc.rawId] || [], supersededBy: [], supersededByTruncated: false };
|
|
159
|
+
results.set(doc.id, result);
|
|
160
|
+
}
|
|
161
|
+
if (result.matches.length < MAX_MATCHES) result.matches.push(match);
|
|
162
|
+
else result.matchesTruncated = true;
|
|
163
|
+
}
|
|
164
|
+
for (const input of inputs) {
|
|
165
|
+
const ancestors = [];
|
|
166
|
+
for (let dir = path.posix.dirname(input); dir !== "."; dir = path.posix.dirname(dir)) ancestors.push(dir);
|
|
167
|
+
for (const repository of scopes) {
|
|
168
|
+
for (const [map, key, kind] of [[index.files, input, "exact_file"], ...[input, ...ancestors].map(dir => [index.directories, dir, "ancestor_directory"])]) {
|
|
169
|
+
for (const posting of map[postingKey(repository, key)] || []) {
|
|
170
|
+
const doc = documents[posting.documentId];
|
|
171
|
+
if (!eligible(doc)) continue;
|
|
172
|
+
counts.get(input).add(doc.id);
|
|
173
|
+
const ref = doc.codeReferences[posting.referenceIndex];
|
|
174
|
+
add(doc, { inputPath: input, kind, recordedPath: ref.path, repository: ref.repository || null, scope: ref.repository ? "explicit" : "unscoped", relation: ref.relation, evidence: { provenance: ref.provenance, bodyLine: ref.bodyLine, excerpt: ref.excerpt } });
|
|
175
|
+
}
|
|
176
|
+
}
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
const byRawId = new Map();
|
|
180
|
+
for (const doc of Object.values(documents)) {
|
|
181
|
+
if (doc.kind === "entry") {
|
|
182
|
+
if (!byRawId.has(doc.rawId)) byRawId.set(doc.rawId, []);
|
|
183
|
+
byRawId.get(doc.rawId).push(doc);
|
|
184
|
+
}
|
|
185
|
+
}
|
|
186
|
+
const edges = new Map();
|
|
187
|
+
let ambiguous = false;
|
|
188
|
+
let inconsistent = false;
|
|
189
|
+
function connect(from, edge) {
|
|
190
|
+
if (!edges.has(from)) edges.set(from, []);
|
|
191
|
+
edges.get(from).push(edge);
|
|
192
|
+
}
|
|
193
|
+
for (const doc of Object.values(documents)) {
|
|
194
|
+
for (const link of doc.links || []) {
|
|
195
|
+
if (!["references", "relates_to", "supersedes"].includes(link.relation)) continue;
|
|
196
|
+
const targets = byRawId.get(link.toEntryId) || [];
|
|
197
|
+
if (targets.length !== 1 || (byRawId.get(doc.rawId) || []).length !== 1) { if (targets.length > 1 || (byRawId.get(doc.rawId) || []).length > 1) ambiguous = true; continue; }
|
|
198
|
+
const target = targets[0];
|
|
199
|
+
if (link.targetPath && path.posix.normalize(path.posix.join(path.posix.dirname(doc.path), link.targetPath)) !== target.path) {
|
|
200
|
+
inconsistent = true;
|
|
201
|
+
continue;
|
|
202
|
+
}
|
|
203
|
+
connect(doc.id, { doc: target, relation: link.relation, direction: "outgoing" });
|
|
204
|
+
connect(target.id, { doc, relation: link.relation, direction: "incoming" });
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
const order = { exact_file: 0, ancestor_directory: 1, linked_context: 2 };
|
|
208
|
+
const rank = entry => Math.min(...entry.matches.map(match => order[match.kind]));
|
|
209
|
+
const sort = (a, b) => rank(a) - rank(b)
|
|
210
|
+
|| Number(b.entryType === "decision") - Number(a.entryType === "decision")
|
|
211
|
+
|| String(b.createdAt).localeCompare(String(a.createdAt)) || a.path.localeCompare(b.path);
|
|
212
|
+
const direct = [...results.values()].sort(sort);
|
|
213
|
+
let examined = 0;
|
|
214
|
+
let relatedTruncated = false;
|
|
215
|
+
outer: for (const source of direct) {
|
|
216
|
+
for (const edge of edges.get(source.documentId) || []) {
|
|
217
|
+
if (examined++ >= MAX_RELATED_EDGES) { relatedTruncated = true; break outer; }
|
|
218
|
+
if (!eligible(edge.doc) || !inScope(edge.doc) || edge.doc.id === source.documentId) continue;
|
|
219
|
+
if (edge.relation !== "supersedes" && !["decision", "doc"].includes(edge.doc.entryType)) continue;
|
|
220
|
+
// Copy the direct routes: cycles never expand newly linked evidence.
|
|
221
|
+
for (const inputPath of new Set(source.matches.filter(m => m.kind !== "linked_context").map(m => m.inputPath))) {
|
|
222
|
+
add(edge.doc, { inputPath, kind: "linked_context", viaDocumentId: source.documentId, viaPath: source.path, relation: edge.relation, direction: edge.direction });
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
}
|
|
226
|
+
for (const result of results.values()) {
|
|
227
|
+
const replacements = (edges.get(result.documentId) || []).filter(e => e.relation === "supersedes" && e.direction === "incoming" && eligible(e.doc) && inScope(e.doc));
|
|
228
|
+
result.supersededByTruncated = replacements.length > MAX_MATCHES;
|
|
229
|
+
result.supersededBy = replacements.slice(0, MAX_MATCHES).map(e => ({ id: e.doc.rawId, documentId: e.doc.id, path: e.doc.path }));
|
|
230
|
+
}
|
|
231
|
+
if (inconsistent) warnings.push({ code: "INCONSISTENT_PATH_EVIDENCE_LINK", detail: "links whose IDs and target paths disagree were not expanded" });
|
|
232
|
+
if (ambiguous) warnings.push({ code: "AMBIGUOUS_PATH_EVIDENCE_LINK", detail: "links with duplicate entry IDs were not expanded" });
|
|
233
|
+
if (Object.values(documents).some(d => d.codeReferencesTruncated)) warnings.push({ code: "CODE_REFERENCES_TRUNCATED", detail: "some entries exceed the reference extraction limit" });
|
|
234
|
+
const all = [...results.values()].sort(sort);
|
|
235
|
+
const entries = all.slice(0, limit);
|
|
236
|
+
return {
|
|
237
|
+
entries,
|
|
238
|
+
totalEntries: all.length,
|
|
239
|
+
truncated: all.length > entries.length,
|
|
240
|
+
relatedTruncated,
|
|
241
|
+
unmatchedPaths: inputs.filter(p => !counts.get(p).size),
|
|
242
|
+
paths: inputs.map(p => ({ path: p, directMatches: counts.get(p).size, returnedDirectMatches: entries.filter(e => counts.get(p).has(e.documentId)).length })),
|
|
243
|
+
warnings,
|
|
244
|
+
};
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
module.exports = { EXTRACTOR_VERSION, extractReferences, buildPathIndex, validPathIndex, normalizeInputs, normalizePath, lookupPaths };
|
package/lib/recall/index.js
CHANGED
|
@@ -4,6 +4,7 @@ const { performance } = require("node:perf_hooks");
|
|
|
4
4
|
const { projectContext } = require("../installer/index.js");
|
|
5
5
|
const { searchIndex } = require("./search.js");
|
|
6
6
|
const { ensureSnapshot } = require("./snapshot.js");
|
|
7
|
+
const { normalizeInputs, lookupPaths } = require("./files.js");
|
|
7
8
|
|
|
8
9
|
function contextFor(options = {}) {
|
|
9
10
|
const target = options.target || process.cwd();
|
|
@@ -53,4 +54,22 @@ function searchRecall(query, options = {}) {
|
|
|
53
54
|
};
|
|
54
55
|
}
|
|
55
56
|
|
|
56
|
-
|
|
57
|
+
function lookupFiles(paths, options = {}) {
|
|
58
|
+
const started = performance.now();
|
|
59
|
+
const context = contextFor(options);
|
|
60
|
+
const inputs = normalizeInputs(paths, { ...options, target: context.target });
|
|
61
|
+
const ensured = ensureSnapshot(context, options);
|
|
62
|
+
const ensureMs = performance.now() - started;
|
|
63
|
+
const results = lookupPaths(ensured.snapshot, inputs, options);
|
|
64
|
+
return {
|
|
65
|
+
action: "recall-files",
|
|
66
|
+
target: context.target,
|
|
67
|
+
projectKey: ensured.snapshot.projectKey,
|
|
68
|
+
repository: options.repository || null,
|
|
69
|
+
cache: cacheSummary(ensured, { ensureMs, totalMs: performance.now() - started }),
|
|
70
|
+
...results,
|
|
71
|
+
warnings: [...ensured.warnings, ...results.warnings],
|
|
72
|
+
};
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
module.exports = { buildRecallIndex, searchRecall, lookupFiles };
|