@fro.bot/wiki-write-core 0.1.2 → 0.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/gate-contract.js +1 -1
- package/dist/gate-contract.json +1 -1
- package/dist/wiki-ingest.d.ts +41 -3
- package/dist/wiki-ingest.js +270 -31
- package/package.json +1 -1
package/dist/gate-contract.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
// Bump deliberately when an intentional consumer-visible gate contract break ships.
|
|
2
2
|
export const GATE_CONTRACT_VERSION = 1;
|
|
3
3
|
// Replaced with the deterministic source-tree hash by the package build.
|
|
4
|
-
export const GATE_SOURCE_TREE_HASH = '
|
|
4
|
+
export const GATE_SOURCE_TREE_HASH = '4c87f6366faeed604a910d0da221bd2d4a9661e9b83ec230c9345829bc0406f2';
|
package/dist/gate-contract.json
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":1,"sourceTreeHash":"
|
|
1
|
+
{"version":1,"sourceTreeHash":"4c87f6366faeed604a910d0da221bd2d4a9661e9b83ec230c9345829bc0406f2"}
|
package/dist/wiki-ingest.d.ts
CHANGED
|
@@ -31,6 +31,16 @@ export interface BuildWikiIngestChangesParams {
|
|
|
31
31
|
trackedMetadataAvailable?: boolean;
|
|
32
32
|
/** System-owned correction state used for post-regeneration survival verification. */
|
|
33
33
|
corrections?: CorrectionsFile;
|
|
34
|
+
/**
|
|
35
|
+
* Log entries an agent appended to its own `knowledge/log.md`, derived by the caller. Appended to
|
|
36
|
+
* the existing log, before this ingest's own summary entry.
|
|
37
|
+
*/
|
|
38
|
+
agentLogEntries?: string;
|
|
39
|
+
/**
|
|
40
|
+
* Pages an agent deleted in a handoff, already removed from `existingFiles`. They join the result's
|
|
41
|
+
* `deletedPaths`, and a call with only deletions needs no page updates.
|
|
42
|
+
*/
|
|
43
|
+
handoffDeletedPaths?: readonly string[];
|
|
34
44
|
}
|
|
35
45
|
export interface BuildWikiIngestChangesResult {
|
|
36
46
|
files: Record<string, string>;
|
|
@@ -38,9 +48,9 @@ export interface BuildWikiIngestChangesResult {
|
|
|
38
48
|
findings: readonly WikiLintFinding[];
|
|
39
49
|
}
|
|
40
50
|
export interface WikiIngestWarning {
|
|
41
|
-
code: 'duplicate-repo-identity' | 'repos-metadata-unavailable';
|
|
51
|
+
code: 'duplicate-repo-identity' | 'repos-metadata-unavailable' | 'agent-log-entries-dropped';
|
|
42
52
|
node_ids?: string[];
|
|
43
|
-
reason?: 'read-failed' | 'parse-failed';
|
|
53
|
+
reason?: 'read-failed' | 'parse-failed' | 'not-append-only';
|
|
44
54
|
slugs?: string[];
|
|
45
55
|
}
|
|
46
56
|
export interface TrackedRepoNodeIdsResult {
|
|
@@ -65,6 +75,21 @@ export interface CommitWikiChangesParams {
|
|
|
65
75
|
* after that read are caught.
|
|
66
76
|
*/
|
|
67
77
|
baseCommitSha?: string;
|
|
78
|
+
/**
|
|
79
|
+
* A second, per-path base for a write whose pages were edited by an agent job in an earlier job.
|
|
80
|
+
* `agentCommitSha` is the data commit that job synced; `paths` are the pages it changed or deleted
|
|
81
|
+
* (`knowledge/log.md` and `knowledge/index.md` are ignored here: the writer rebuilds both itself).
|
|
82
|
+
*
|
|
83
|
+
* Before any write, the commit must resolve and be an ancestor of the branch head, or the call fails
|
|
84
|
+
* with `HANDOFF_BASE_INVALID`. Every attempt then compares each path on the head it reads with the
|
|
85
|
+
* same path in that commit's tree, ahead of the `baseCommitSha` check, and fails with
|
|
86
|
+
* `HANDOFF_PAGE_CHANGED` if any differs: another writer edited or deleted the page since the agent
|
|
87
|
+
* read it.
|
|
88
|
+
*/
|
|
89
|
+
handoff?: {
|
|
90
|
+
agentCommitSha: string;
|
|
91
|
+
paths: readonly string[];
|
|
92
|
+
};
|
|
68
93
|
octokit?: OctokitClient;
|
|
69
94
|
maxRetries?: number;
|
|
70
95
|
/**
|
|
@@ -82,13 +107,15 @@ export interface RunWikiIngestCliDependencies {
|
|
|
82
107
|
readonly readCorrections?: typeof readCorrections;
|
|
83
108
|
readonly commitWikiChanges?: typeof commitWikiChanges;
|
|
84
109
|
readonly getChangedWikiPaths?: typeof getChangedWikiPaths;
|
|
110
|
+
/** Client for reading the agent's base commit; defaults to one built from `GITHUB_TOKEN`. */
|
|
111
|
+
readonly octokit?: OctokitClient;
|
|
85
112
|
}
|
|
86
113
|
/**
|
|
87
114
|
* Narrow Octokit client type derived from the real `@octokit/rest` SDK.
|
|
88
115
|
* See commit-metadata.ts for the rationale behind deriving rather than handwriting.
|
|
89
116
|
*/
|
|
90
117
|
export type OctokitClient = Octokit;
|
|
91
|
-
export type WikiIngestErrorCode = 'INVALID_PAYLOAD' | 'INVALID_PAGE_PATH' | 'INVALID_FRONTMATTER' | 'INVALID_WIKILINK' | 'CORRECTION_ERODED' | 'INVALID_RETRIES' | 'PROTECTED_BRANCH' | 'MISSING_TOKEN' | 'OCTOKIT_LOAD_FAILED' | 'CONFLICT_EXHAUSTED';
|
|
118
|
+
export type WikiIngestErrorCode = 'INVALID_PAYLOAD' | 'INVALID_PAGE_PATH' | 'INVALID_FRONTMATTER' | 'INVALID_WIKILINK' | 'CORRECTION_ERODED' | 'INVALID_RETRIES' | 'PROTECTED_BRANCH' | 'MISSING_TOKEN' | 'OCTOKIT_LOAD_FAILED' | 'CONFLICT_EXHAUSTED' | 'HANDOFF_BASE_INVALID' | 'HANDOFF_PAGE_CHANGED';
|
|
92
119
|
export declare class WikiIngestError extends Error {
|
|
93
120
|
readonly code: WikiIngestErrorCode;
|
|
94
121
|
readonly remediation: string;
|
|
@@ -100,6 +127,8 @@ export declare class WikiIngestError extends Error {
|
|
|
100
127
|
findings?: readonly WikiLintFinding[];
|
|
101
128
|
});
|
|
102
129
|
}
|
|
130
|
+
/** The two wiki files every writer shares. Writers rebuild both from their own sync; a handoff never supplies them. */
|
|
131
|
+
export declare function isSharedWikiFile(path: string): boolean;
|
|
103
132
|
export declare function buildWikiIngestChanges(params: BuildWikiIngestChangesParams): BuildWikiIngestChangesResult;
|
|
104
133
|
/**
|
|
105
134
|
* Commits `params.files` (and removes `params.deletedPaths`) as one commit on the data branch.
|
|
@@ -109,6 +138,9 @@ export declare function buildWikiIngestChanges(params: BuildWikiIngestChangesPar
|
|
|
109
138
|
* when it is omitted), another writer changed it, so the write fails with `CONFLICT_EXHAUSTED` and
|
|
110
139
|
* nothing is written. A ref race (`updateRef` 409, or 422 not-a-fast-forward) re-reads the head and
|
|
111
140
|
* replays the same files on it, up to `maxRetries` attempts, under the same guard.
|
|
141
|
+
*
|
|
142
|
+
* With `handoff`, the handed-off paths are checked first on every attempt against the agent's data
|
|
143
|
+
* commit (`HANDOFF_PAGE_CHANGED`), so a page conflict is never reported as a ref race.
|
|
112
144
|
*/
|
|
113
145
|
export declare function commitWikiChanges(params: CommitWikiChangesParams): Promise<CommitWikiChangesResult>;
|
|
114
146
|
export declare function validateWikilinks(files: Record<string, string>): void;
|
|
@@ -182,6 +214,12 @@ export declare function parsePorcelainPaths(stdout: string): string[];
|
|
|
182
214
|
* the four category directories.
|
|
183
215
|
*/
|
|
184
216
|
export declare function countWikiPages(paths: string[]): number;
|
|
217
|
+
/**
|
|
218
|
+
* The log text an agent appended: what its `knowledge/log.md` adds after `knowledge/log.md` at the
|
|
219
|
+
* commit it started from. `undefined` when the agent changed anything before that point, or the base
|
|
220
|
+
* log is empty, so there is nothing safe to take. Both inputs are normalized first (LF, one final newline).
|
|
221
|
+
*/
|
|
222
|
+
export declare function deriveAgentLogEntries(baseLog: string, handoffLog: string): string | undefined;
|
|
185
223
|
export declare function runWikiIngestCli(dependencies?: RunWikiIngestCliDependencies): Promise<void>;
|
|
186
224
|
type ReadUtf8File = (path: string, encoding: 'utf8') => Promise<string>;
|
|
187
225
|
export declare function loadTrackedRepoNodeIds(readFileImpl?: ReadUtf8File): Promise<TrackedRepoNodeIdsResult>;
|
package/dist/wiki-ingest.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
+
import { Buffer } from 'node:buffer';
|
|
1
2
|
import { execFile } from 'node:child_process';
|
|
2
3
|
import { appendFile, readdir, readFile } from 'node:fs/promises';
|
|
3
|
-
import { basename } from 'node:path';
|
|
4
|
+
import { basename, join } from 'node:path';
|
|
4
5
|
import process from 'node:process';
|
|
5
6
|
import { promisify } from 'node:util';
|
|
6
7
|
import { parse, stringify } from 'yaml';
|
|
@@ -19,6 +20,8 @@ const WIKI_ROOT = 'knowledge/wiki';
|
|
|
19
20
|
const INDEX_PATH = 'knowledge/index.md';
|
|
20
21
|
const LOG_PATH = 'knowledge/log.md';
|
|
21
22
|
const DATE_PATTERN = /^\d{4}-\d{2}-\d{2}$/;
|
|
23
|
+
const COMMIT_SHA_PATTERN = /^[0-9a-f]{40}(?:[0-9a-f]{24})?$/u;
|
|
24
|
+
const HANDOFF_BASE_REMEDIATION = 'The writer could not tie the handoff to the data commit the agent job synced. Re-run the workflow so the agent job runs again.';
|
|
22
25
|
const execFileAsync = promisify(execFile);
|
|
23
26
|
export class WikiIngestError extends Error {
|
|
24
27
|
code;
|
|
@@ -32,8 +35,14 @@ export class WikiIngestError extends Error {
|
|
|
32
35
|
this.findings = params.findings ?? [];
|
|
33
36
|
}
|
|
34
37
|
}
|
|
38
|
+
/** The two wiki files every writer shares. Writers rebuild both from their own sync; a handoff never supplies them. */
|
|
39
|
+
export function isSharedWikiFile(path) {
|
|
40
|
+
return path === INDEX_PATH || path === LOG_PATH;
|
|
41
|
+
}
|
|
35
42
|
export function buildWikiIngestChanges(params) {
|
|
36
|
-
if (params.pages.length === 0
|
|
43
|
+
if (params.pages.length === 0 &&
|
|
44
|
+
(params.handoffDeletedPaths ?? []).length === 0 &&
|
|
45
|
+
(params.agentLogEntries ?? '').trim() === '') {
|
|
37
46
|
throw new WikiIngestError({
|
|
38
47
|
code: 'INVALID_PAYLOAD',
|
|
39
48
|
message: 'wiki ingest requires at least one page update',
|
|
@@ -51,6 +60,10 @@ export function buildWikiIngestChanges(params) {
|
|
|
51
60
|
nextFiles[page.path] = normalized;
|
|
52
61
|
files[page.path] = normalized;
|
|
53
62
|
}
|
|
63
|
+
for (const path of params.handoffDeletedPaths ?? []) {
|
|
64
|
+
if (nextFiles[path] === undefined && !deletedPaths.includes(path))
|
|
65
|
+
deletedPaths.push(path);
|
|
66
|
+
}
|
|
54
67
|
rewriteInboundWikilinks(nextFiles, migrations);
|
|
55
68
|
for (const [path, content] of Object.entries(nextFiles)) {
|
|
56
69
|
if (params.existingFiles[path] !== content && !deletedPaths.includes(path)) {
|
|
@@ -93,7 +106,7 @@ export function buildWikiIngestChanges(params) {
|
|
|
93
106
|
}
|
|
94
107
|
const parsedPages = collectWikiPages(nextFiles);
|
|
95
108
|
const index = buildIndexDocument(nextFiles[INDEX_PATH], parsedPages);
|
|
96
|
-
const log = appendLogEntry(nextFiles[LOG_PATH], params);
|
|
109
|
+
const log = appendLogEntry(appendAgentLogEntries(nextFiles[LOG_PATH], params.agentLogEntries), params);
|
|
97
110
|
files[INDEX_PATH] = index;
|
|
98
111
|
files[LOG_PATH] = log;
|
|
99
112
|
return { files, deletedPaths, findings: correctionFindings };
|
|
@@ -129,6 +142,9 @@ function collectWikiFrontmatter(files) {
|
|
|
129
142
|
* when it is omitted), another writer changed it, so the write fails with `CONFLICT_EXHAUSTED` and
|
|
130
143
|
* nothing is written. A ref race (`updateRef` 409, or 422 not-a-fast-forward) re-reads the head and
|
|
131
144
|
* replays the same files on it, up to `maxRetries` attempts, under the same guard.
|
|
145
|
+
*
|
|
146
|
+
* With `handoff`, the handed-off paths are checked first on every attempt against the agent's data
|
|
147
|
+
* commit (`HANDOFF_PAGE_CHANGED`), so a page conflict is never reported as a ref race.
|
|
132
148
|
*/
|
|
133
149
|
export async function commitWikiChanges(params) {
|
|
134
150
|
const owner = params.owner ?? DEFAULT_OWNER;
|
|
@@ -144,6 +160,12 @@ export async function commitWikiChanges(params) {
|
|
|
144
160
|
}
|
|
145
161
|
const octokit = params.octokit ?? (await createOctokitFromEnv());
|
|
146
162
|
rejectProtectedWikiBranchName(branch);
|
|
163
|
+
// Resolved outside the retry loop's try, ahead of any bootstrap or write: an unusable agent commit is
|
|
164
|
+
// terminal, and the loop's 404 handler must not mistake it for a missing data branch.
|
|
165
|
+
const handoffBaseTreeSha = params.handoff === undefined
|
|
166
|
+
? undefined
|
|
167
|
+
: await resolveHandoffBaseTree(octokit, owner, repo, branch, params.handoff.agentCommitSha);
|
|
168
|
+
const handoffPaths = (params.handoff?.paths ?? []).filter(path => !isSharedWikiFile(path));
|
|
147
169
|
const shouldBootstrapDataBranch = branch === DEFAULT_BRANCH;
|
|
148
170
|
const bootstrap = params.bootstrapDataBranch ?? defaultBootstrapDataBranch;
|
|
149
171
|
const bootstrapDataBranch = async () => {
|
|
@@ -167,23 +189,37 @@ export async function commitWikiChanges(params) {
|
|
|
167
189
|
params.baseCommitSha === undefined
|
|
168
190
|
? commit.data.tree.sha
|
|
169
191
|
: await readCommitTreeSha(octokit, owner, repo, params.baseCommitSha);
|
|
192
|
+
const location = `${owner}/${repo}@${branch}`;
|
|
193
|
+
// Handed-off pages first, against the commit the agent read, so a page conflict surfaces as such
|
|
194
|
+
// even when the head also moved under the shared files.
|
|
195
|
+
if (handoffBaseTreeSha !== undefined && handoffPaths.length > 0 && commit.data.tree.sha !== handoffBaseTreeSha) {
|
|
196
|
+
await assertWriteSetUnchanged(octokit, owner, repo, {
|
|
197
|
+
baseTreeSha: handoffBaseTreeSha,
|
|
198
|
+
headTreeSha: commit.data.tree.sha,
|
|
199
|
+
paths: handoffPaths,
|
|
200
|
+
location,
|
|
201
|
+
code: 'HANDOFF_PAGE_CHANGED',
|
|
202
|
+
});
|
|
203
|
+
}
|
|
170
204
|
const guardBaseTreeSha = baseTreeSha;
|
|
171
205
|
if (commit.data.tree.sha !== guardBaseTreeSha) {
|
|
172
206
|
await assertWriteSetUnchanged(octokit, owner, repo, {
|
|
173
207
|
baseTreeSha: guardBaseTreeSha,
|
|
174
208
|
headTreeSha: commit.data.tree.sha,
|
|
175
209
|
paths: [...Object.keys(params.files), ...(params.deletedPaths ?? [])],
|
|
176
|
-
location
|
|
210
|
+
location,
|
|
211
|
+
code: 'CONFLICT_EXHAUSTED',
|
|
177
212
|
});
|
|
178
213
|
}
|
|
179
214
|
const tree = [];
|
|
215
|
+
// Read before the first write: a deletion of a path that cannot be confirmed must not be skipped.
|
|
216
|
+
const presentPaths = params.deletedPaths === undefined || params.deletedPaths.length === 0
|
|
217
|
+
? new Set()
|
|
218
|
+
: await getPresentPathsInTree(octokit, owner, repo, commit.data.tree.sha);
|
|
180
219
|
for (const [path, content] of Object.entries(params.files)) {
|
|
181
220
|
const blob = await octokit.rest.git.createBlob({ owner, repo, content, encoding: 'utf-8' });
|
|
182
221
|
tree.push({ path, mode: '100644', type: 'blob', sha: blob.data.sha });
|
|
183
222
|
}
|
|
184
|
-
const presentPaths = params.deletedPaths === undefined || params.deletedPaths.length === 0
|
|
185
|
-
? new Set()
|
|
186
|
-
: await getPresentPathsInTree(octokit, owner, repo, commit.data.tree.sha);
|
|
187
223
|
for (const path of params.deletedPaths ?? []) {
|
|
188
224
|
if (params.files[path] !== undefined || !presentPaths.has(path)) {
|
|
189
225
|
continue;
|
|
@@ -256,40 +292,110 @@ async function assertWritableWikiBranch(octokit, owner, repo, branch) {
|
|
|
256
292
|
});
|
|
257
293
|
}
|
|
258
294
|
async function getPresentPathsInTree(octokit, owner, repo, treeSha) {
|
|
259
|
-
const
|
|
260
|
-
if (
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
295
|
+
const entries = await readTreeEntries(octokit, owner, repo, treeSha);
|
|
296
|
+
if (entries === undefined)
|
|
297
|
+
throw truncatedTreeRefusal('a deletion cannot be checked against a truncated tree');
|
|
298
|
+
return new Set(entries.keys());
|
|
299
|
+
}
|
|
300
|
+
/** A truncated tree listing cannot be trusted either way, so the write fails closed. */
|
|
301
|
+
function truncatedTreeRefusal(reason) {
|
|
302
|
+
return new WikiIngestError({
|
|
303
|
+
code: 'CONFLICT_EXHAUSTED',
|
|
304
|
+
message: `wiki ingest refused: ${reason}`,
|
|
305
|
+
remediation: 'The data branch tree listing was truncated. Retry the workflow; if it persists, the tree has outgrown the recursive listing.',
|
|
306
|
+
});
|
|
266
307
|
}
|
|
267
308
|
async function readCommitTreeSha(octokit, owner, repo, sha) {
|
|
268
309
|
const commit = await octokit.rest.git.getCommit({ owner, repo, commit_sha: sha });
|
|
269
310
|
return commit.data.tree.sha;
|
|
270
311
|
}
|
|
271
312
|
/**
|
|
272
|
-
*
|
|
273
|
-
*
|
|
274
|
-
*
|
|
313
|
+
* Resolves the agent job's data commit to its tree, after checking it is a full commit SHA and an
|
|
314
|
+
* ancestor of (or equal to) the branch head. Compare's `behind_by` is 0 and its merge base is the
|
|
315
|
+
* commit itself exactly when that holds.
|
|
316
|
+
*/
|
|
317
|
+
async function resolveHandoffBaseTree(octokit, owner, repo, branch, agentCommitSha) {
|
|
318
|
+
if (!COMMIT_SHA_PATTERN.test(agentCommitSha)) {
|
|
319
|
+
throw handoffBaseInvalid('the agent job data commit is missing or is not a commit SHA');
|
|
320
|
+
}
|
|
321
|
+
try {
|
|
322
|
+
const head = await octokit.rest.git.getRef({ owner, repo, ref: `heads/${branch}` });
|
|
323
|
+
const comparison = await octokit.rest.repos.compareCommitsWithBasehead({
|
|
324
|
+
owner,
|
|
325
|
+
repo,
|
|
326
|
+
basehead: `${agentCommitSha}...${head.data.object.sha}`,
|
|
327
|
+
});
|
|
328
|
+
if (comparison.data.behind_by !== 0 || comparison.data.merge_base_commit.sha !== agentCommitSha) {
|
|
329
|
+
throw handoffBaseInvalid(`the agent job data commit is not an ancestor of ${owner}/${repo}@${branch}`);
|
|
330
|
+
}
|
|
331
|
+
return await readCommitTreeSha(octokit, owner, repo, agentCommitSha);
|
|
332
|
+
}
|
|
333
|
+
catch (error) {
|
|
334
|
+
if (error instanceof WikiIngestError)
|
|
335
|
+
throw error;
|
|
336
|
+
if (isApiErrorStatus(error, 404) || isApiErrorStatus(error, 422)) {
|
|
337
|
+
throw handoffBaseInvalid(`the agent job data commit does not resolve on ${owner}/${repo}@${branch}`);
|
|
338
|
+
}
|
|
339
|
+
throw error;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
/**
|
|
343
|
+
* Refuses, before any content is validated, a handoff whose pages another writer has changed since the
|
|
344
|
+
* agent job synced. Validating a stale page against the current wiki can fail for an unrelated reason
|
|
345
|
+
* (a link to a page the other writer deleted), which would hide the conflict. The commit repeats this
|
|
346
|
+
* check on every attempt, which still catches a change that lands after this one.
|
|
347
|
+
*/
|
|
348
|
+
async function assertHandoffPagesCurrent(octokit, owner, repo, branch, handoff) {
|
|
349
|
+
const baseTreeSha = await resolveHandoffBaseTree(octokit, owner, repo, branch, handoff.agentCommitSha);
|
|
350
|
+
const paths = handoff.paths.filter(path => !isSharedWikiFile(path));
|
|
351
|
+
if (paths.length === 0)
|
|
352
|
+
return;
|
|
353
|
+
const head = await octokit.rest.git.getRef({ owner, repo, ref: `heads/${branch}` });
|
|
354
|
+
const headTreeSha = await readCommitTreeSha(octokit, owner, repo, head.data.object.sha);
|
|
355
|
+
if (headTreeSha === baseTreeSha)
|
|
356
|
+
return;
|
|
357
|
+
await assertWriteSetUnchanged(octokit, owner, repo, {
|
|
358
|
+
baseTreeSha,
|
|
359
|
+
headTreeSha,
|
|
360
|
+
paths,
|
|
361
|
+
location: `${owner}/${repo}@${branch}`,
|
|
362
|
+
code: 'HANDOFF_PAGE_CHANGED',
|
|
363
|
+
});
|
|
364
|
+
}
|
|
365
|
+
/**
|
|
366
|
+
* Fail closed unless every path has the same tree entry (sha, mode, type; absence counts) on the new
|
|
367
|
+
* head as in the tree the files were derived from. A replay onto a head where another writer touched
|
|
368
|
+
* one of them would silently overwrite that writer's change. `code` names the refusal; a truncated
|
|
369
|
+
* tree read always refuses with `CONFLICT_EXHAUSTED`, because it says nothing about the pages.
|
|
275
370
|
*/
|
|
276
371
|
async function assertWriteSetUnchanged(octokit, owner, repo, params) {
|
|
277
|
-
const refuse = (reason) => {
|
|
372
|
+
const refuse = (code, reason) => {
|
|
278
373
|
throw new WikiIngestError({
|
|
279
|
-
code
|
|
280
|
-
message:
|
|
281
|
-
|
|
374
|
+
code,
|
|
375
|
+
message: code === 'HANDOFF_PAGE_CHANGED'
|
|
376
|
+
? `wiki ingest refused to write to ${params.location}: ${reason}`
|
|
377
|
+
: `wiki ingest did not replay onto ${params.location}: ${reason}`,
|
|
378
|
+
remediation: code === 'HANDOFF_PAGE_CHANGED'
|
|
379
|
+
? 'Another writer changed a page after the agent job synced. Re-run the agent job so it starts from the current data branch.'
|
|
380
|
+
: 'Another writer updated the data branch concurrently. Retry the workflow so it rebuilds on the new head.',
|
|
282
381
|
});
|
|
283
382
|
};
|
|
284
383
|
const base = await readTreeEntries(octokit, owner, repo, params.baseTreeSha);
|
|
285
384
|
const head = await readTreeEntries(octokit, owner, repo, params.headTreeSha);
|
|
286
385
|
if (base === undefined || head === undefined)
|
|
287
|
-
return refuse('a tree read was truncated, so the write set cannot be verified');
|
|
288
|
-
if (params.paths.some(path => base.get(path)
|
|
289
|
-
return refuse(
|
|
386
|
+
return refuse('CONFLICT_EXHAUSTED', 'a tree read was truncated, so the write set cannot be verified');
|
|
387
|
+
if (params.paths.some(path => !sameTreeEntry(base.get(path), head.get(path)))) {
|
|
388
|
+
return refuse(params.code, params.code === 'HANDOFF_PAGE_CHANGED'
|
|
389
|
+
? 'a handed-off page changed after the agent job synced'
|
|
390
|
+
: 'another writer changed a path in the write set');
|
|
290
391
|
}
|
|
291
392
|
}
|
|
292
|
-
|
|
393
|
+
function sameTreeEntry(left, right) {
|
|
394
|
+
if (left === undefined || right === undefined)
|
|
395
|
+
return left === right;
|
|
396
|
+
return left.mode === right.mode && left.type === right.type && left.sha === right.sha;
|
|
397
|
+
}
|
|
398
|
+
/** Path to its entry, for every non-directory entry. `undefined` when the read is truncated. */
|
|
293
399
|
async function readTreeEntries(octokit, owner, repo, treeSha) {
|
|
294
400
|
const response = await octokit.rest.git.getTree({ owner, repo, tree_sha: treeSha, recursive: 'true' });
|
|
295
401
|
if (response.data.truncated === true)
|
|
@@ -301,7 +407,7 @@ async function readTreeEntries(octokit, owner, repo, treeSha) {
|
|
|
301
407
|
for (const entry of tree) {
|
|
302
408
|
if (!isRecord(entry) || typeof entry.path !== 'string' || entry.type === 'tree')
|
|
303
409
|
continue;
|
|
304
|
-
entries.set(entry.path,
|
|
410
|
+
entries.set(entry.path, { mode: String(entry.mode), type: String(entry.type), sha: String(entry.sha) });
|
|
305
411
|
}
|
|
306
412
|
return entries;
|
|
307
413
|
}
|
|
@@ -533,6 +639,13 @@ function parseIndexEntryLines(index) {
|
|
|
533
639
|
}
|
|
534
640
|
return entries;
|
|
535
641
|
}
|
|
642
|
+
/** Appends raw log entries (already in log format) to `existingLog`, keeping its text byte-for-byte. */
|
|
643
|
+
function appendAgentLogEntries(existingLog, entries) {
|
|
644
|
+
if (entries === undefined || entries.trim() === '')
|
|
645
|
+
return existingLog;
|
|
646
|
+
const base = existingLog === undefined || existingLog === '' ? LOG_HEADER : normalizeText(existingLog);
|
|
647
|
+
return normalizeText(`${base}${entries.startsWith('\n') ? entries : `\n${entries}`}`);
|
|
648
|
+
}
|
|
536
649
|
/**
|
|
537
650
|
* Append one entry to `knowledge/log.md`, keeping the existing text byte-for-byte. The log is
|
|
538
651
|
* append-only; callers must never rewrite history. Exported for the operator rename path.
|
|
@@ -935,7 +1048,7 @@ async function getChangedWikiPaths() {
|
|
|
935
1048
|
`${WIKI_ROOT}/entities`,
|
|
936
1049
|
`${WIKI_ROOT}/comparisons`,
|
|
937
1050
|
]);
|
|
938
|
-
return parsePorcelainPaths(stdout).filter(path => path
|
|
1051
|
+
return parsePorcelainPaths(stdout).filter(path => !isSharedWikiFile(path) && path.endsWith('.md'));
|
|
939
1052
|
}
|
|
940
1053
|
/**
|
|
941
1054
|
* Parse paths from `git status --porcelain=v1` output.
|
|
@@ -1031,6 +1144,114 @@ async function emitPagesChanged(n) {
|
|
|
1031
1144
|
await appendFile(outputPath, `pages_changed=${n}\n`);
|
|
1032
1145
|
}
|
|
1033
1146
|
}
|
|
1147
|
+
function envValue(name) {
|
|
1148
|
+
const value = process.env[name];
|
|
1149
|
+
return value === undefined || value === '' ? undefined : value;
|
|
1150
|
+
}
|
|
1151
|
+
function handoffBaseInvalid(message) {
|
|
1152
|
+
return new WikiIngestError({
|
|
1153
|
+
code: 'HANDOFF_BASE_INVALID',
|
|
1154
|
+
message: `wiki ingest refused: ${message}`,
|
|
1155
|
+
remediation: HANDOFF_BASE_REMEDIATION,
|
|
1156
|
+
});
|
|
1157
|
+
}
|
|
1158
|
+
/**
|
|
1159
|
+
* LF line endings and exactly one final newline; an empty log stays empty. Unlike `normalizeText`, which
|
|
1160
|
+
* only adds a missing final newline, this also converts CRLF and drops trailing blank lines.
|
|
1161
|
+
*/
|
|
1162
|
+
function toLfWithSingleFinalNewline(value) {
|
|
1163
|
+
const trimmed = value.replaceAll('\r\n', '\n').trimEnd();
|
|
1164
|
+
return trimmed === '' ? '' : `${trimmed}\n`;
|
|
1165
|
+
}
|
|
1166
|
+
/**
|
|
1167
|
+
* The log text an agent appended: what its `knowledge/log.md` adds after `knowledge/log.md` at the
|
|
1168
|
+
* commit it started from. `undefined` when the agent changed anything before that point, or the base
|
|
1169
|
+
* log is empty, so there is nothing safe to take. Both inputs are normalized first (LF, one final newline).
|
|
1170
|
+
*/
|
|
1171
|
+
export function deriveAgentLogEntries(baseLog, handoffLog) {
|
|
1172
|
+
const base = toLfWithSingleFinalNewline(baseLog);
|
|
1173
|
+
const handoff = toLfWithSingleFinalNewline(handoffLog);
|
|
1174
|
+
if (base === '' || !handoff.startsWith(base))
|
|
1175
|
+
return undefined;
|
|
1176
|
+
return handoff.slice(base.length);
|
|
1177
|
+
}
|
|
1178
|
+
async function readHandoffManifest(handoffDir) {
|
|
1179
|
+
const raw = await readFile(join(handoffDir, 'manifest.json'), 'utf8');
|
|
1180
|
+
let parsed;
|
|
1181
|
+
try {
|
|
1182
|
+
parsed = JSON.parse(raw);
|
|
1183
|
+
}
|
|
1184
|
+
catch {
|
|
1185
|
+
throw new WikiIngestError({
|
|
1186
|
+
code: 'INVALID_PAYLOAD',
|
|
1187
|
+
message: 'wiki handoff manifest.json is not valid JSON',
|
|
1188
|
+
remediation: 'Rebuild the handoff artifact with scripts/wiki-handoff-build.ts.',
|
|
1189
|
+
});
|
|
1190
|
+
}
|
|
1191
|
+
const isStringArray = (value) => Array.isArray(value) && value.every(entry => typeof entry === 'string');
|
|
1192
|
+
if (!isRecord(parsed) || !isStringArray(parsed.changed) || !isStringArray(parsed.deleted)) {
|
|
1193
|
+
throw new WikiIngestError({
|
|
1194
|
+
code: 'INVALID_PAYLOAD',
|
|
1195
|
+
message: 'wiki handoff manifest.json must hold string arrays "changed" and "deleted"',
|
|
1196
|
+
remediation: 'Rebuild the handoff artifact with scripts/wiki-handoff-build.ts.',
|
|
1197
|
+
});
|
|
1198
|
+
}
|
|
1199
|
+
return { changed: parsed.changed, deleted: parsed.deleted };
|
|
1200
|
+
}
|
|
1201
|
+
/** The text of `path` in the tree of `commitSha`, or `undefined` when the commit has no such file. */
|
|
1202
|
+
async function readFileAtCommit(octokit, owner, repo, commitSha, path) {
|
|
1203
|
+
const entries = await readTreeEntries(octokit, owner, repo, await readCommitTreeSha(octokit, owner, repo, commitSha));
|
|
1204
|
+
if (entries === undefined)
|
|
1205
|
+
throw truncatedTreeRefusal(`${path} cannot be read from a truncated tree`);
|
|
1206
|
+
const entry = entries.get(path);
|
|
1207
|
+
if (entry === undefined || entry.type !== 'blob')
|
|
1208
|
+
return undefined;
|
|
1209
|
+
const blob = await octokit.rest.git.getBlob({ owner, repo, file_sha: entry.sha });
|
|
1210
|
+
return Buffer.from(blob.data.content, 'base64').toString('utf8');
|
|
1211
|
+
}
|
|
1212
|
+
/**
|
|
1213
|
+
* Reads the handoff the trusted job applied. `undefined` outside a handoff run (neither
|
|
1214
|
+
* `WIKI_HANDOFF_DIR` nor `WIKI_HANDOFF_BASE_SHA` set). Inside one, the agent job's data commit must be a
|
|
1215
|
+
* full commit SHA, the handed-off pages must be unchanged on the branch since that commit (checked here,
|
|
1216
|
+
* ahead of any validation of their content), and the log entries the agent appended are derived against that commit's `log.md`.
|
|
1217
|
+
*/
|
|
1218
|
+
async function loadHandoff(dependencies, target) {
|
|
1219
|
+
const handoffDir = envValue('WIKI_HANDOFF_DIR');
|
|
1220
|
+
const agentCommitSha = envValue('WIKI_HANDOFF_BASE_SHA');
|
|
1221
|
+
if (handoffDir === undefined && agentCommitSha === undefined)
|
|
1222
|
+
return undefined;
|
|
1223
|
+
if (handoffDir === undefined || agentCommitSha === undefined || !COMMIT_SHA_PATTERN.test(agentCommitSha)) {
|
|
1224
|
+
throw handoffBaseInvalid('the agent job data commit is missing or is not a commit SHA');
|
|
1225
|
+
}
|
|
1226
|
+
const octokit = dependencies.octokit ?? (await createOctokitFromEnv());
|
|
1227
|
+
const manifest = await readHandoffManifest(handoffDir);
|
|
1228
|
+
const pagePaths = [...manifest.changed, ...manifest.deleted];
|
|
1229
|
+
const deletedPagePaths = manifest.deleted.filter(path => !isSharedWikiFile(path));
|
|
1230
|
+
await assertHandoffPagesCurrent(octokit, target.owner ?? DEFAULT_OWNER, target.repo ?? DEFAULT_REPO, target.branch ?? DEFAULT_BRANCH, { agentCommitSha, paths: pagePaths });
|
|
1231
|
+
if (!manifest.changed.includes(LOG_PATH)) {
|
|
1232
|
+
return { octokit, agentCommitSha, agentLogEntries: undefined, pagePaths, deletedPagePaths, warnings: [] };
|
|
1233
|
+
}
|
|
1234
|
+
const handoffLog = await readFile(join(handoffDir, 'files', LOG_PATH), 'utf8');
|
|
1235
|
+
let baseLog;
|
|
1236
|
+
try {
|
|
1237
|
+
baseLog = await readFileAtCommit(octokit, target.owner ?? DEFAULT_OWNER, target.repo ?? DEFAULT_REPO, agentCommitSha, LOG_PATH);
|
|
1238
|
+
}
|
|
1239
|
+
catch (error) {
|
|
1240
|
+
// Only a commit or file that does not exist is a bad base; any other failure is not about the base.
|
|
1241
|
+
if (isApiErrorStatus(error, 404))
|
|
1242
|
+
throw handoffBaseInvalid(`the agent job data commit does not resolve`);
|
|
1243
|
+
throw error;
|
|
1244
|
+
}
|
|
1245
|
+
if (baseLog === undefined)
|
|
1246
|
+
throw handoffBaseInvalid(`${LOG_PATH} does not exist at the agent job data commit`);
|
|
1247
|
+
const agentLogEntries = deriveAgentLogEntries(baseLog, handoffLog);
|
|
1248
|
+
const warnings = agentLogEntries === undefined ? [{ code: 'agent-log-entries-dropped', reason: 'not-append-only' }] : [];
|
|
1249
|
+
return { octokit, agentCommitSha, agentLogEntries, pagePaths, deletedPagePaths, warnings };
|
|
1250
|
+
}
|
|
1251
|
+
function writeWarnings(warnings) {
|
|
1252
|
+
for (const warning of warnings)
|
|
1253
|
+
process.stderr.write(`wiki-ingest:warning:${JSON.stringify(warning)}\n`);
|
|
1254
|
+
}
|
|
1034
1255
|
export async function runWikiIngestCli(dependencies = {}) {
|
|
1035
1256
|
const payloadPath = process.argv[2] ?? process.env.WIKI_INGEST_INPUT;
|
|
1036
1257
|
const readCorrectionsImpl = dependencies.readCorrections ?? readCorrections;
|
|
@@ -1062,9 +1283,12 @@ export async function runWikiIngestCli(dependencies = {}) {
|
|
|
1062
1283
|
let message;
|
|
1063
1284
|
let identityWarnings = [];
|
|
1064
1285
|
let committedPagePaths;
|
|
1286
|
+
let handoff;
|
|
1065
1287
|
try {
|
|
1066
1288
|
if (payloadPath !== undefined && payloadPath !== '') {
|
|
1067
1289
|
const payload = parsePayload(await readFile(payloadPath, 'utf8'));
|
|
1290
|
+
handoff = await loadHandoff(dependencies, payload);
|
|
1291
|
+
writeWarnings(handoff?.warnings ?? []);
|
|
1068
1292
|
const tracked = await loadTrackedRepoNodeIds();
|
|
1069
1293
|
identityWarnings = tracked.warnings;
|
|
1070
1294
|
built = buildWikiIngestChanges({
|
|
@@ -1079,6 +1303,8 @@ export async function runWikiIngestCli(dependencies = {}) {
|
|
|
1079
1303
|
fallbackNodeId: payload.node_id,
|
|
1080
1304
|
trackedMetadataAvailable: tracked.metadataAvailable,
|
|
1081
1305
|
corrections: correctionRead.corrections,
|
|
1306
|
+
agentLogEntries: handoff?.agentLogEntries,
|
|
1307
|
+
handoffDeletedPaths: handoff?.deletedPagePaths,
|
|
1082
1308
|
});
|
|
1083
1309
|
owner = payload.owner;
|
|
1084
1310
|
repo = payload.repo;
|
|
@@ -1088,7 +1314,17 @@ export async function runWikiIngestCli(dependencies = {}) {
|
|
|
1088
1314
|
}
|
|
1089
1315
|
else {
|
|
1090
1316
|
const changedPaths = await getChangedWikiPathsImpl();
|
|
1091
|
-
|
|
1317
|
+
handoff = await loadHandoff(dependencies, {
|
|
1318
|
+
owner: process.env.WIKI_OWNER,
|
|
1319
|
+
repo: process.env.WIKI_REPO,
|
|
1320
|
+
branch: process.env.WIKI_BRANCH,
|
|
1321
|
+
});
|
|
1322
|
+
writeWarnings(handoff?.warnings ?? []);
|
|
1323
|
+
// `git status` drops deletions, so an agent's deleted pages come from the manifest. A handoff
|
|
1324
|
+
// whose only change is appended log entries still commits the log and the rebuilt index.
|
|
1325
|
+
if (changedPaths.length === 0 &&
|
|
1326
|
+
(handoff?.deletedPagePaths ?? []).length === 0 &&
|
|
1327
|
+
(handoff?.agentLogEntries ?? '').trim() === '') {
|
|
1092
1328
|
await emitPagesChanged(0);
|
|
1093
1329
|
process.stdout.write(`${JSON.stringify({ committed: false, attempts: 1, pagesChanged: 0 })}\n`);
|
|
1094
1330
|
return;
|
|
@@ -1110,6 +1346,8 @@ export async function runWikiIngestCli(dependencies = {}) {
|
|
|
1110
1346
|
targetNodeId: process.env.REPO_NODE_ID,
|
|
1111
1347
|
trackedMetadataAvailable: tracked.metadataAvailable,
|
|
1112
1348
|
corrections: correctionRead.corrections,
|
|
1349
|
+
agentLogEntries: handoff?.agentLogEntries,
|
|
1350
|
+
handoffDeletedPaths: handoff?.deletedPagePaths,
|
|
1113
1351
|
});
|
|
1114
1352
|
owner = process.env.WIKI_OWNER;
|
|
1115
1353
|
repo = process.env.WIKI_REPO;
|
|
@@ -1128,9 +1366,7 @@ export async function runWikiIngestCli(dependencies = {}) {
|
|
|
1128
1366
|
}
|
|
1129
1367
|
throw error;
|
|
1130
1368
|
}
|
|
1131
|
-
|
|
1132
|
-
process.stderr.write(`wiki-ingest:warning:${JSON.stringify(warning)}\n`);
|
|
1133
|
-
}
|
|
1369
|
+
writeWarnings(identityWarnings);
|
|
1134
1370
|
for (const finding of built.findings) {
|
|
1135
1371
|
process.stderr.write(`wiki-ingest:finding:${JSON.stringify(finding)}\n`);
|
|
1136
1372
|
}
|
|
@@ -1142,6 +1378,9 @@ export async function runWikiIngestCli(dependencies = {}) {
|
|
|
1142
1378
|
files: built.files,
|
|
1143
1379
|
deletedPaths: built.deletedPaths,
|
|
1144
1380
|
...(baseCommitSha === undefined ? {} : { baseCommitSha }),
|
|
1381
|
+
...(handoff === undefined
|
|
1382
|
+
? {}
|
|
1383
|
+
: { octokit: handoff.octokit, handoff: { agentCommitSha: handoff.agentCommitSha, paths: handoff.pagePaths } }),
|
|
1145
1384
|
});
|
|
1146
1385
|
const pagesChanged = countWikiPages(committedPagePaths);
|
|
1147
1386
|
await emitPagesChanged(pagesChanged);
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@fro.bot/wiki-write-core",
|
|
3
|
-
"version": "0.1.
|
|
3
|
+
"version": "0.1.3",
|
|
4
4
|
"description": "Shared validation and commit primitives for Fro Bot wiki writes",
|
|
5
5
|
"homepage": "https://github.com/fro-bot/.github/tree/main/packages/wiki-write-core#readme",
|
|
6
6
|
"bugs": {
|