@dogfood-lab/ingest 1.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +81 -0
- package/lib/atomic-write.js +101 -0
- package/lib/rename-with-retry.js +39 -0
- package/lib/unsafe-segment.js +36 -0
- package/load-context.js +131 -0
- package/package.json +57 -0
- package/persist.js +172 -0
- package/rebuild-indexes.js +356 -0
- package/run.js +540 -0
- package/validate-record.js +83 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 mcp-tool-shop
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<a href="https://github.com/dogfood-lab/testing-os">
|
|
3
|
+
<img src="https://raw.githubusercontent.com/dogfood-lab/testing-os/main/assets/logo.png" alt="testing-os" width="280">
|
|
4
|
+
</a>
|
|
5
|
+
</p>
|
|
6
|
+
|
|
7
|
+
# @dogfood-lab/ingest
|
|
8
|
+
|
|
9
|
+
> Ingestion pipeline for testing-os. Thin glue: dispatch → verifier → persist → indexes.
|
|
10
|
+
|
|
11
|
+
Part of the [`testing-os`](https://github.com/dogfood-lab/testing-os) monorepo — the operating system for testing in the AI era.
|
|
12
|
+
|
|
13
|
+
Runs on the receiving side of the dogfood loop. Receives `repository_dispatch` payloads from consumer repos, validates them through `@dogfood-lab/verify`, persists records under `records/`, and rebuilds the read-side indexes (`latest-by-repo.json`, `failing.json`, `stale.json`) atomically.
|
|
14
|
+
|
|
15
|
+
## Install
|
|
16
|
+
|
|
17
|
+
```bash
|
|
18
|
+
npm install @dogfood-lab/ingest
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
## Usage — programmatic
|
|
22
|
+
|
|
23
|
+
```js
|
|
24
|
+
import { run } from '@dogfood-lab/ingest';
|
|
25
|
+
|
|
26
|
+
const result = await run({
|
|
27
|
+
payloadPath: './incoming.json',
|
|
28
|
+
repoRoot: process.cwd(),
|
|
29
|
+
provenance: 'github',
|
|
30
|
+
});
|
|
31
|
+
|
|
32
|
+
console.log(result.persisted_records); // count of new records written
|
|
33
|
+
console.log(result.indexes_rebuilt); // boolean
|
|
34
|
+
console.log(result.rejected); // array of rejection reasons (if any)
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
## Usage — CLI
|
|
38
|
+
|
|
39
|
+
```bash
|
|
40
|
+
npx @dogfood-lab/ingest run --payload incoming.json --repo-root .
|
|
41
|
+
```
|
|
42
|
+
|
|
43
|
+
The CLI exits with structured exit codes:
|
|
44
|
+
|
|
45
|
+
- `0` — ok, all records persisted
|
|
46
|
+
- `1` — user error (bad payload shape, missing files, invalid args)
|
|
47
|
+
- `2` — runtime error (downstream validator failure, I/O failure)
|
|
48
|
+
- `3` — partial success (some records persisted, some rejected)
|
|
49
|
+
|
|
50
|
+
## Pipeline stages
|
|
51
|
+
|
|
52
|
+
| Stage | Module | Output |
|
|
53
|
+
|---|---|---|
|
|
54
|
+
| 1. Load context | `load-context.js` | Reads existing `records/`, `policies/`, prior indexes into memory |
|
|
55
|
+
| 2. Validate | delegates to `@dogfood-lab/verify` | Verdict per record: `ok` or `rejection_reasons[]` |
|
|
56
|
+
| 3. Persist | `persist.js` | Two-phase atomic write to `records/<repo>/<record-id>.yaml` |
|
|
57
|
+
| 4. Rebuild indexes | `rebuild-indexes.js` | Regenerates `latest-by-repo.json`, `failing.json`, `stale.json` with crash-safe journaling |
|
|
58
|
+
|
|
59
|
+
Each stage emits a structured event via `lib/log-stage.js` so the ingest loop is observable end-to-end. Failures at any stage carry a `code`, `message`, and `hint` per the testing-os structured error contract.
|
|
60
|
+
|
|
61
|
+
## Concurrency + crash safety
|
|
62
|
+
|
|
63
|
+
- **Per-repo locking**: advisory file lock via `@dogfood-lab/findings/lib/file-lock.js` — Windows-compatible (uses `linkSync` CAS, not `flock`).
|
|
64
|
+
- **Atomic two-phase commit**: `lib/atomic-write.js` writes to a `.tmp-<pid>.json` shadow first, then renames into place with `renameWithRetry` for Windows AV scanner handle-release windows.
|
|
65
|
+
- **Idempotent crash recovery**: `cleanupCrashedJournals` runs at every entry point; partial writes from previous crashes are detected and either completed or rolled back.
|
|
66
|
+
|
|
67
|
+
## What testing-os ingest does NOT touch
|
|
68
|
+
|
|
69
|
+
- Consumer source code beyond what's referenced in the dispatch envelope
|
|
70
|
+
- Secrets beyond the dispatch payload's own fields
|
|
71
|
+
- Anything outside the testing-os repo's working tree
|
|
72
|
+
|
|
73
|
+
The receiver workflow runs with `contents: write` scoped to the testing-os repo only.
|
|
74
|
+
|
|
75
|
+
## Docs
|
|
76
|
+
|
|
77
|
+
📖 Full handbook: **<https://dogfood-lab.github.io/testing-os/handbook/>**
|
|
78
|
+
|
|
79
|
+
## License
|
|
80
|
+
|
|
81
|
+
MIT © 2026 mcp-tool-shop
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Atomic file write — temp+rename so torn writes never silently drop a file
|
|
3
|
+
* from listings. `rename` is atomic on POSIX and Windows: a concurrent reader
|
|
4
|
+
* sees either the old contents or the new contents — never a half-written file.
|
|
5
|
+
*
|
|
6
|
+
* Sibling of `packages/findings/lib/atomic-write.js`. The two files have the
|
|
7
|
+
* same contract; the duplication exists because npm workspaces do not allow
|
|
8
|
+
* `ingest → findings` imports (findings already depends on ingest, and adding
|
|
9
|
+
* the reverse edge would create a workspace dependency cycle). Class #6
|
|
10
|
+
* helper-adoption-sweep treats them as a single canonical pattern even though
|
|
11
|
+
* the source lives in two places — the canonical contract is "one writeFileSync
|
|
12
|
+
* to a temp suffix, then renameSync; no caller assembles the temp+rename
|
|
13
|
+
* pattern inline anywhere else under packages/ingest/."
|
|
14
|
+
*
|
|
15
|
+
* Used by `rebuild-indexes.js` for the per-file leg of its multi-file commit
|
|
16
|
+
* group (W3-PIPE-002). The commit group itself is a thin layer on top of
|
|
17
|
+
* this helper — see `commitGroupRename` in rebuild-indexes for that.
|
|
18
|
+
*/
|
|
19
|
+
|
|
20
|
+
import fs from 'node:fs';
|
|
21
|
+
import { randomBytes } from 'node:crypto';
|
|
22
|
+
import { renameWithRetry } from './rename-with-retry.js';
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Atomically write `content` to `path`.
|
|
26
|
+
*
|
|
27
|
+
* @param {string} path - Final destination path.
|
|
28
|
+
* @param {string} content - File contents.
|
|
29
|
+
* @param {BufferEncoding} [encoding='utf-8'] - Write encoding.
|
|
30
|
+
* @returns {string} - The path that was written.
|
|
31
|
+
*/
|
|
32
|
+
export function atomicWriteFileSync(path, content, encoding = 'utf-8') {
|
|
33
|
+
const tmpSuffix = randomBytes(4).toString('hex');
|
|
34
|
+
const tmpPath = `${path}.${tmpSuffix}.tmp`;
|
|
35
|
+
try {
|
|
36
|
+
// Indirect via `fs.*` (not destructured imports) so test mocks of
|
|
37
|
+
// `fs.writeFileSync` reach this code path. Mirrors findings/lib/atomic-write.js.
|
|
38
|
+
fs.writeFileSync(tmpPath, content, encoding);
|
|
39
|
+
// Windows-only: rename can transiently fail with EPERM/EBUSY when AV or
|
|
40
|
+
// Search Indexer holds a handle on the just-written temp. Bounded retry.
|
|
41
|
+
renameWithRetry(tmpPath, path);
|
|
42
|
+
} catch (err) {
|
|
43
|
+
try { fs.unlinkSync(tmpPath); } catch { /* tmp may not exist */ }
|
|
44
|
+
throw err;
|
|
45
|
+
}
|
|
46
|
+
return path;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
/**
|
|
50
|
+
* Stage `content` to a temp file alongside `path`. Returns the temp path so
|
|
51
|
+
* a later commit-group operation can rename it. Does NOT remove the temp
|
|
52
|
+
* if the caller throws — caller owns cleanup via `discardStaged`.
|
|
53
|
+
*
|
|
54
|
+
* Why exposed: `rebuild-indexes.js` needs a two-phase-commit shape — write
|
|
55
|
+
* all 3 temps, THEN rename them all in sequence. The non-staging
|
|
56
|
+
* `atomicWriteFileSync` couples write+rename, which is exactly what we
|
|
57
|
+
* cannot do for multi-file atomicity.
|
|
58
|
+
*
|
|
59
|
+
* @param {string} path - Final destination path (NOT created here).
|
|
60
|
+
* @param {string} content
|
|
61
|
+
* @param {BufferEncoding} [encoding='utf-8']
|
|
62
|
+
* @returns {string} The temp path holding the staged content.
|
|
63
|
+
*/
|
|
64
|
+
export function stageWriteFileSync(path, content, encoding = 'utf-8') {
|
|
65
|
+
const tmpSuffix = randomBytes(4).toString('hex');
|
|
66
|
+
const tmpPath = `${path}.${tmpSuffix}.tmp`;
|
|
67
|
+
try {
|
|
68
|
+
fs.writeFileSync(tmpPath, content, encoding);
|
|
69
|
+
return tmpPath;
|
|
70
|
+
} catch (err) {
|
|
71
|
+
try { fs.unlinkSync(tmpPath); } catch { /* tmp may not exist */ }
|
|
72
|
+
throw err;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Promote one staged temp path to its final name. Wraps `renameSync` so
|
|
78
|
+
* call sites don't have to know the rename is the only step. Throws on
|
|
79
|
+
* failure; partial-failure handling is the caller's responsibility (see
|
|
80
|
+
* `rebuild-indexes.js`'s `commitGroupRename` for the multi-file case).
|
|
81
|
+
*
|
|
82
|
+
* @param {string} tmpPath
|
|
83
|
+
* @param {string} finalPath
|
|
84
|
+
*/
|
|
85
|
+
export function promoteStaged(tmpPath, finalPath) {
|
|
86
|
+
// Windows EPERM/EBUSY tolerance — the multi-file commit group in
|
|
87
|
+
// rebuild-indexes.js relies on each promoteStaged hop succeeding; a
|
|
88
|
+
// transient AV/Search-Indexer handle would otherwise abort the group
|
|
89
|
+
// partway through.
|
|
90
|
+
renameWithRetry(tmpPath, finalPath);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
/**
|
|
94
|
+
* Best-effort cleanup of staged temp files. Safe to call on a path that
|
|
95
|
+
* doesn't exist.
|
|
96
|
+
*
|
|
97
|
+
* @param {string} tmpPath
|
|
98
|
+
*/
|
|
99
|
+
export function discardStaged(tmpPath) {
|
|
100
|
+
try { fs.unlinkSync(tmpPath); } catch { /* may not exist */ }
|
|
101
|
+
}
|
|
@@ -0,0 +1,39 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* renameWithRetry — Windows-tolerant atomic rename.
|
|
3
|
+
*
|
|
4
|
+
* Sibling of `packages/findings/lib/rename-with-retry.js`. The two files
|
|
5
|
+
* have the same contract; the duplication exists because npm workspaces do
|
|
6
|
+
* not allow `ingest → findings` imports (findings already depends on ingest,
|
|
7
|
+
* and adding the reverse edge would create a workspace dependency cycle).
|
|
8
|
+
*
|
|
9
|
+
* On Windows NTFS, `renameSync(tmp, target)` can throw EPERM/EBUSY even when
|
|
10
|
+
* the calling process holds the file lock, because antivirus / Search Indexer
|
|
11
|
+
* / backup agents hold transient handles across the rename window. Bounded
|
|
12
|
+
* exponential backoff retries past the transient handle.
|
|
13
|
+
*
|
|
14
|
+
* Synchronous-on-purpose: the call sites (atomic-write helpers, rebuild-indexes,
|
|
15
|
+
* event-log appender) are themselves synchronous, and an async retry would
|
|
16
|
+
* leak the boundary into otherwise-deterministic flush paths.
|
|
17
|
+
*
|
|
18
|
+
* @param {string} tmp - Source path (the just-written temp file).
|
|
19
|
+
* @param {string} dest - Destination path (the canonical artifact).
|
|
20
|
+
* @param {object} [opts]
|
|
21
|
+
* @param {number} [opts.retries=10] - Max retry attempts after the first failure.
|
|
22
|
+
* @param {number} [opts.baseMs=15] - Initial backoff in ms (doubles each step).
|
|
23
|
+
* @param {number} [opts.maxMs=200] - Cap on per-step backoff.
|
|
24
|
+
*/
|
|
25
|
+
import { renameSync } from 'node:fs';
|
|
26
|
+
|
|
27
|
+
export function renameWithRetry(tmp, dest, { retries = 10, baseMs = 15, maxMs = 200 } = {}) {
|
|
28
|
+
for (let i = 0; i <= retries; i++) {
|
|
29
|
+
try {
|
|
30
|
+
renameSync(tmp, dest);
|
|
31
|
+
return;
|
|
32
|
+
} catch (err) {
|
|
33
|
+
if ((err.code !== 'EPERM' && err.code !== 'EBUSY') || i === retries) throw err;
|
|
34
|
+
const delay = Math.min(baseMs * (1 << i), maxMs);
|
|
35
|
+
const until = Date.now() + delay;
|
|
36
|
+
while (Date.now() < until) { /* spin */ }
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* unsafe-segment.js — central path-segment safety helper.
|
|
3
|
+
*
|
|
4
|
+
* Three callsites previously defined or duplicated this regex (F-916867-005):
|
|
5
|
+
* - packages/ingest/persist.js (canonical instance)
|
|
6
|
+
* - packages/ingest/load-context.js (loadRepoPolicy + githubScenarioFetcher)
|
|
7
|
+
* - packages/findings/derive/load-records.js (the missing third callsite)
|
|
8
|
+
*
|
|
9
|
+
* The check rejects path-traversal substrings (`..`) and any path separator
|
|
10
|
+
* (`/`, `\`). Single dots remain legal because GitHub permits dotted org/repo
|
|
11
|
+
* names like `next.js`, `mcp-tool-shop.github.io`, `repo.io`. The submission
|
|
12
|
+
* schema's repo pattern `^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$` agrees.
|
|
13
|
+
*
|
|
14
|
+
* F-375053-006 regression — an earlier `/[.\/]/` was over-broad and crashed
|
|
15
|
+
* legitimate submissions inside writeRecord. The narrower `/\.\.|[/\\]/` has
|
|
16
|
+
* stood since wave 9; this helper is the productized form.
|
|
17
|
+
*/
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Regex matching unsafe substrings in a single path segment.
|
|
21
|
+
* Use `.test(segment)` — returns true if the segment is unsafe.
|
|
22
|
+
*
|
|
23
|
+
* @type {RegExp}
|
|
24
|
+
*/
|
|
25
|
+
export const UNSAFE_SEGMENT = /\.\.|[/\\]/;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Predicate form: returns true when the given segment contains a path-traversal
|
|
29
|
+
* substring or a path separator.
|
|
30
|
+
*
|
|
31
|
+
* @param {string} segment - A single path-segment candidate (e.g. an org or repo name).
|
|
32
|
+
* @returns {boolean}
|
|
33
|
+
*/
|
|
34
|
+
export function isUnsafeSegment(segment) {
|
|
35
|
+
return UNSAFE_SEGMENT.test(segment);
|
|
36
|
+
}
|
package/load-context.js
ADDED
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Context loader
|
|
3
|
+
*
|
|
4
|
+
* Gathers everything the verifier needs:
|
|
5
|
+
* - Global policy
|
|
6
|
+
* - Repo policy (optional, missing is valid)
|
|
7
|
+
* - Scenario definitions from source repo (optional, missing becomes rejection reason)
|
|
8
|
+
* - Payload normalization
|
|
9
|
+
*
|
|
10
|
+
* Scenario loading uses a fetch adapter so it can be stubbed in tests.
|
|
11
|
+
*/
|
|
12
|
+
|
|
13
|
+
import { readFileSync, existsSync } from 'node:fs';
|
|
14
|
+
import { join } from 'node:path';
|
|
15
|
+
import yaml from 'js-yaml';
|
|
16
|
+
|
|
17
|
+
import { isUnsafeSegment } from './lib/unsafe-segment.js';
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* Load the global policy.
|
|
21
|
+
*
|
|
22
|
+
* @param {string} repoRoot
|
|
23
|
+
* @returns {object}
|
|
24
|
+
*/
|
|
25
|
+
export function loadGlobalPolicy(repoRoot) {
|
|
26
|
+
const path = join(repoRoot, 'policies', 'global-policy.yaml');
|
|
27
|
+
return yaml.load(readFileSync(path, 'utf-8'));
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Load repo-specific policy. Returns null if no policy exists.
|
|
32
|
+
*
|
|
33
|
+
* @param {string} repoSlug - e.g. "mcp-tool-shop-org/dogfood-labs"
|
|
34
|
+
* @param {string} repoRoot
|
|
35
|
+
* @returns {object|null}
|
|
36
|
+
*/
|
|
37
|
+
export function loadRepoPolicy(repoSlug, repoRoot) {
|
|
38
|
+
const [org, repo] = repoSlug.split('/');
|
|
39
|
+
if (!org || !repo || isUnsafeSegment(org) || isUnsafeSegment(repo)) return null;
|
|
40
|
+
const path = join(repoRoot, 'policies', 'repos', org, `${repo}.yaml`);
|
|
41
|
+
|
|
42
|
+
if (!existsSync(path)) return null;
|
|
43
|
+
try {
|
|
44
|
+
return yaml.load(readFileSync(path, 'utf-8'));
|
|
45
|
+
} catch {
|
|
46
|
+
console.warn(`load-context: malformed YAML in repo policy for ${repoSlug}`);
|
|
47
|
+
return null;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
|
|
51
|
+
/**
|
|
52
|
+
* Default scenario fetcher that reads from the local filesystem.
|
|
53
|
+
* Used when dogfood-labs is dogfooding itself.
|
|
54
|
+
*
|
|
55
|
+
* @param {string} repoRoot - Root of the source repo
|
|
56
|
+
* @returns {object} Scenario fetch adapter
|
|
57
|
+
*/
|
|
58
|
+
export function localScenarioFetcher(repoRoot) {
|
|
59
|
+
return {
|
|
60
|
+
async fetch(scenarioId) {
|
|
61
|
+
if (!/^[\w-]+$/.test(scenarioId)) return null;
|
|
62
|
+
const path = join(repoRoot, 'dogfood', 'scenarios', `${scenarioId}.yaml`);
|
|
63
|
+
if (!existsSync(path)) return null;
|
|
64
|
+
return yaml.load(readFileSync(path, 'utf-8'));
|
|
65
|
+
}
|
|
66
|
+
};
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
/**
|
|
70
|
+
* GitHub scenario fetcher. Loads scenario definitions from a source repo
|
|
71
|
+
* via the GitHub API at a specific commit SHA.
|
|
72
|
+
*
|
|
73
|
+
* @param {string} token - GitHub PAT
|
|
74
|
+
* @param {string} repoSlug - e.g. "mcp-tool-shop-org/shipcheck"
|
|
75
|
+
* @param {string} commitSha - Commit to fetch scenarios from
|
|
76
|
+
* @returns {object} Scenario fetch adapter
|
|
77
|
+
*/
|
|
78
|
+
export function githubScenarioFetcher(token, repoSlug, commitSha) {
|
|
79
|
+
const [org, repo] = repoSlug.split('/');
|
|
80
|
+
if (!org || !repo || isUnsafeSegment(org) || isUnsafeSegment(repo)) {
|
|
81
|
+
return { async fetch() { return null; } };
|
|
82
|
+
}
|
|
83
|
+
return {
|
|
84
|
+
async fetch(scenarioId) {
|
|
85
|
+
if (!/^[\w-]+$/.test(scenarioId)) return null;
|
|
86
|
+
const path = `dogfood/scenarios/${scenarioId}.yaml`;
|
|
87
|
+
const url = `https://api.github.com/repos/${repoSlug}/contents/${path}?ref=${commitSha}`;
|
|
88
|
+
|
|
89
|
+
try {
|
|
90
|
+
const resp = await globalThis.fetch(url, {
|
|
91
|
+
headers: {
|
|
92
|
+
Authorization: `Bearer ${token}`,
|
|
93
|
+
Accept: 'application/vnd.github.raw+json',
|
|
94
|
+
'X-GitHub-Api-Version': '2022-11-28'
|
|
95
|
+
}
|
|
96
|
+
});
|
|
97
|
+
if (!resp.ok) return null;
|
|
98
|
+
const text = await resp.text();
|
|
99
|
+
return yaml.load(text);
|
|
100
|
+
} catch {
|
|
101
|
+
return null;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
};
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/**
|
|
108
|
+
* Load all scenario definitions referenced by a submission's scenario_results.
|
|
109
|
+
*
|
|
110
|
+
* @param {object} submission
|
|
111
|
+
* @param {object} scenarioFetcher - { fetch(scenarioId) => Promise<object|null> }
|
|
112
|
+
* @returns {Promise<{ scenarios: Map<string, object>, errors: string[] }>}
|
|
113
|
+
*/
|
|
114
|
+
export async function loadScenarios(submission, scenarioFetcher) {
|
|
115
|
+
const scenarios = new Map();
|
|
116
|
+
const errors = [];
|
|
117
|
+
|
|
118
|
+
for (const sr of submission.scenario_results || []) {
|
|
119
|
+
const id = sr.scenario_id;
|
|
120
|
+
if (scenarios.has(id)) continue;
|
|
121
|
+
|
|
122
|
+
const definition = await scenarioFetcher.fetch(id);
|
|
123
|
+
if (definition) {
|
|
124
|
+
scenarios.set(id, definition);
|
|
125
|
+
} else {
|
|
126
|
+
errors.push(`scenario "${id}" could not be loaded from source repo`);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
return { scenarios, errors };
|
|
131
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@dogfood-lab/ingest",
|
|
3
|
+
"version": "1.2.1",
|
|
4
|
+
"type": "module",
|
|
5
|
+
"description": "Ingestion pipeline for testing-os. Thin glue: dispatch → verifier → persist → indexes.",
|
|
6
|
+
"main": "run.js",
|
|
7
|
+
"exports": {
|
|
8
|
+
".": "./run.js",
|
|
9
|
+
"./lib/*": "./lib/*"
|
|
10
|
+
},
|
|
11
|
+
"scripts": {
|
|
12
|
+
"test": "node --test",
|
|
13
|
+
"ingest": "node run.js"
|
|
14
|
+
},
|
|
15
|
+
"files": [
|
|
16
|
+
"run.js",
|
|
17
|
+
"persist.js",
|
|
18
|
+
"rebuild-indexes.js",
|
|
19
|
+
"validate-record.js",
|
|
20
|
+
"load-context.js",
|
|
21
|
+
"lib/",
|
|
22
|
+
"README.md",
|
|
23
|
+
"LICENSE"
|
|
24
|
+
],
|
|
25
|
+
"publishConfig": {
|
|
26
|
+
"access": "public"
|
|
27
|
+
},
|
|
28
|
+
"dependencies": {
|
|
29
|
+
"@dogfood-lab/dogfood-swarm": "^1.2.0",
|
|
30
|
+
"@dogfood-lab/schemas": "^1.2.0",
|
|
31
|
+
"@dogfood-lab/verify": "^1.2.0",
|
|
32
|
+
"ajv": "^8.18.0",
|
|
33
|
+
"ajv-formats": "^3.0.1",
|
|
34
|
+
"js-yaml": "^4.1.0"
|
|
35
|
+
},
|
|
36
|
+
"engines": {
|
|
37
|
+
"node": ">=20"
|
|
38
|
+
},
|
|
39
|
+
"author": "mcp-tool-shop",
|
|
40
|
+
"license": "MIT",
|
|
41
|
+
"repository": {
|
|
42
|
+
"type": "git",
|
|
43
|
+
"url": "https://github.com/dogfood-lab/testing-os.git",
|
|
44
|
+
"directory": "packages/ingest"
|
|
45
|
+
},
|
|
46
|
+
"homepage": "https://github.com/dogfood-lab/testing-os",
|
|
47
|
+
"bugs": {
|
|
48
|
+
"url": "https://github.com/dogfood-lab/testing-os/issues"
|
|
49
|
+
},
|
|
50
|
+
"keywords": [
|
|
51
|
+
"testing-os",
|
|
52
|
+
"dogfood-lab",
|
|
53
|
+
"ingest",
|
|
54
|
+
"pipeline",
|
|
55
|
+
"verification"
|
|
56
|
+
]
|
|
57
|
+
}
|
package/persist.js
ADDED
|
@@ -0,0 +1,172 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Persist layer
|
|
3
|
+
*
|
|
4
|
+
* Writes verified records to the canonical sharded path.
|
|
5
|
+
* Handles: accepted/rejected routing, atomic write (temp+rename),
|
|
6
|
+
* duplicate detection by run_id, directory creation.
|
|
7
|
+
*/
|
|
8
|
+
|
|
9
|
+
import { existsSync, mkdirSync, writeFileSync, renameSync, openSync, closeSync, unlinkSync } from 'node:fs';
|
|
10
|
+
import { join, dirname } from 'node:path';
|
|
11
|
+
import { randomBytes } from 'node:crypto';
|
|
12
|
+
|
|
13
|
+
import { validateRecord } from './validate-record.js';
|
|
14
|
+
import { isUnsafeSegment } from './lib/unsafe-segment.js';
|
|
15
|
+
|
|
16
|
+
/**
|
|
17
|
+
* Error thrown when writeRecord loses a TOCTOU race for the same canonical path.
|
|
18
|
+
* The first concurrent writer wins; the loser sees this error.
|
|
19
|
+
*/
|
|
20
|
+
export class DuplicateRunIdError extends Error {
|
|
21
|
+
constructor(runId, path) {
|
|
22
|
+
super(`duplicate run_id: ${runId} — another writer won the race for ${path}`);
|
|
23
|
+
this.name = 'DuplicateRunIdError';
|
|
24
|
+
this.code = 'DUPLICATE_RUN_ID';
|
|
25
|
+
this.runId = runId;
|
|
26
|
+
this.path = path;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* Compute the canonical file path for a persisted record.
|
|
32
|
+
*
|
|
33
|
+
* Accepted: records/<org>/<repo>/YYYY/MM/DD/run-<run_id>.json
|
|
34
|
+
* Rejected: records/_rejected/<org>/<repo>/YYYY/MM/DD/run-<run_id>.json
|
|
35
|
+
*
|
|
36
|
+
* @param {object} record - Persisted record
|
|
37
|
+
* @param {string} repoRoot - Absolute path to dogfood-labs repo root
|
|
38
|
+
* @returns {string} Absolute file path
|
|
39
|
+
*/
|
|
40
|
+
export function computeRecordPath(record, repoRoot) {
|
|
41
|
+
const status = record.verification?.status;
|
|
42
|
+
const base = status === 'rejected' ? 'records/_rejected' : 'records';
|
|
43
|
+
|
|
44
|
+
const [org, repo] = (record.repo || '').split('/');
|
|
45
|
+
if (!org || !repo) {
|
|
46
|
+
throw new Error(`invalid repo format: ${record.repo}`);
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
// Path-traversal guard: reject `..` substrings and any path separator.
|
|
50
|
+
// Single dots are legal in GitHub org/repo names (e.g. `next.js`,
|
|
51
|
+
// `mcp-tool-shop.github.io`) and the submission schema's repo pattern
|
|
52
|
+
// `^[a-zA-Z0-9_.-]+/[a-zA-Z0-9_.-]+$` allows them. Centralized in
|
|
53
|
+
// ./lib/unsafe-segment.js so all three callsites (persist, load-context,
|
|
54
|
+
// findings/derive/load-records) agree by import — F-916867-005.
|
|
55
|
+
if (isUnsafeSegment(org) || isUnsafeSegment(repo)) {
|
|
56
|
+
throw new Error(`unsafe repo segment: ${record.repo}`);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
if (!/^[\w-]+$/.test(record.run_id)) {
|
|
60
|
+
throw new Error(`unsafe run_id: ${record.run_id}`);
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
const finishedAt = record.timing?.finished_at;
|
|
64
|
+
if (!finishedAt) {
|
|
65
|
+
throw new Error('record missing timing.finished_at');
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const date = new Date(finishedAt);
|
|
69
|
+
if (isNaN(date.getTime())) {
|
|
70
|
+
throw new Error('Invalid finished_at timestamp');
|
|
71
|
+
}
|
|
72
|
+
const year = String(date.getUTCFullYear());
|
|
73
|
+
const month = String(date.getUTCMonth() + 1).padStart(2, '0');
|
|
74
|
+
const day = String(date.getUTCDate()).padStart(2, '0');
|
|
75
|
+
|
|
76
|
+
const filename = `run-${record.run_id}.json`;
|
|
77
|
+
|
|
78
|
+
return join(repoRoot, base, org, repo, year, month, day, filename);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Check if a record with this run_id already exists (accepted or rejected).
|
|
83
|
+
*
|
|
84
|
+
* @param {string} runId
|
|
85
|
+
* @param {object} record - The record (used for repo/timing to compute path)
|
|
86
|
+
* @param {string} repoRoot
|
|
87
|
+
* @returns {boolean}
|
|
88
|
+
*/
|
|
89
|
+
export function isDuplicate(runId, record, repoRoot) {
|
|
90
|
+
// Check accepted path
|
|
91
|
+
const acceptedRecord = { ...record, verification: { ...record.verification, status: 'accepted' } };
|
|
92
|
+
const acceptedPath = computeRecordPath(acceptedRecord, repoRoot);
|
|
93
|
+
if (existsSync(acceptedPath)) return true;
|
|
94
|
+
|
|
95
|
+
// Check rejected path
|
|
96
|
+
const rejectedRecord = { ...record, verification: { ...record.verification, status: 'rejected' } };
|
|
97
|
+
const rejectedPath = computeRecordPath(rejectedRecord, repoRoot);
|
|
98
|
+
if (existsSync(rejectedPath)) return true;
|
|
99
|
+
|
|
100
|
+
return false;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Write a record atomically: write to temp file, then exclusive-rename into place.
|
|
105
|
+
*
|
|
106
|
+
* Race semantics: the canonical path is created via `open(path, 'wx')` (exclusive
|
|
107
|
+
* create — fails if the path exists). Two concurrent ingests for the same run_id
|
|
108
|
+
* can both pass `isDuplicate` (no file yet); the FIRST `open(wx)` wins and the
|
|
109
|
+
* SECOND throws `DuplicateRunIdError` instead of silently overwriting. The
|
|
110
|
+
* temp+rename pattern still provides crash-atomicity — the canonical file is
|
|
111
|
+
* either fully written or absent, never partial.
|
|
112
|
+
*
|
|
113
|
+
* Why not just `existsSync` then `writeFileSync`? That's the original race —
|
|
114
|
+
* the existsSync check and the write are not atomic. `open(wx)` collapses both
|
|
115
|
+
* into a single OS-level call.
|
|
116
|
+
*
|
|
117
|
+
* @param {object} record - Persisted record
|
|
118
|
+
* @param {string} repoRoot - Absolute path to dogfood-labs repo root
|
|
119
|
+
* @returns {{ path: string, written: boolean }} path and whether a write occurred
|
|
120
|
+
* @throws {DuplicateRunIdError} when a concurrent writer won the race
|
|
121
|
+
*/
|
|
122
|
+
export function writeRecord(record, repoRoot) {
|
|
123
|
+
if (isDuplicate(record.run_id, record, repoRoot)) {
|
|
124
|
+
const path = computeRecordPath(record, repoRoot);
|
|
125
|
+
return { path, written: false };
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
// Enforce dogfood-record.schema.json BEFORE touching the filesystem.
|
|
129
|
+
// Better to throw loudly than silently persist a malformed record — the
|
|
130
|
+
// schema is the contract every downstream consumer relies on.
|
|
131
|
+
validateRecord(record);
|
|
132
|
+
|
|
133
|
+
const path = computeRecordPath(record, repoRoot);
|
|
134
|
+
const dir = dirname(path);
|
|
135
|
+
|
|
136
|
+
mkdirSync(dir, { recursive: true });
|
|
137
|
+
|
|
138
|
+
// Race-safe atomic create: try to claim the canonical path with O_EXCL first.
|
|
139
|
+
// If another writer already won the race, fail closed with DuplicateRunIdError
|
|
140
|
+
// — never silently overwrite. On success, hold an empty file we'll fill via
|
|
141
|
+
// temp+rename so the visible bytes are still atomic.
|
|
142
|
+
let claimed = false;
|
|
143
|
+
try {
|
|
144
|
+
const fd = openSync(path, 'wx');
|
|
145
|
+
closeSync(fd);
|
|
146
|
+
claimed = true;
|
|
147
|
+
} catch (err) {
|
|
148
|
+
if (err && err.code === 'EEXIST') {
|
|
149
|
+
throw new DuplicateRunIdError(record.run_id, path);
|
|
150
|
+
}
|
|
151
|
+
throw err;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// Atomic write: temp file → rename over the empty placeholder.
|
|
155
|
+
const tmpSuffix = randomBytes(4).toString('hex');
|
|
156
|
+
const tmpPath = `${path}.${tmpSuffix}.tmp`;
|
|
157
|
+
|
|
158
|
+
try {
|
|
159
|
+
writeFileSync(tmpPath, JSON.stringify(record, null, 2) + '\n', 'utf-8');
|
|
160
|
+
renameSync(tmpPath, path);
|
|
161
|
+
} catch (err) {
|
|
162
|
+
// On any failure after we claimed the path, release the claim so a retry
|
|
163
|
+
// can succeed. The tmp file is best-effort cleanup.
|
|
164
|
+
if (claimed) {
|
|
165
|
+
try { unlinkSync(path); } catch { /* placeholder already gone */ }
|
|
166
|
+
}
|
|
167
|
+
try { unlinkSync(tmpPath); } catch { /* tmp may not exist */ }
|
|
168
|
+
throw err;
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
return { path, written: true };
|
|
172
|
+
}
|