@shidesheng0218/dsh-plugin-lint 0.0.0-stage → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/NOTICE +13 -0
- package/README.md +734 -2
- package/bin/dsh-plugin-analyze.mjs +378 -0
- package/bin/dsh-plugin-lint.mjs +381 -0
- package/bin/dsh-plugin-matrix.mjs +78 -0
- package/bin/dsh-plugin-upgrade-plan.mjs +180 -0
- package/package.json +62 -4
- package/schema/dsh-manifest.schema.json +134 -0
- package/scripts/lib/probe.mjs +34 -0
- package/scripts/probe-boot.mjs +123 -0
- package/src/composition.js +1 -0
- package/src/core/baseline.js +98 -0
- package/src/core/changed.js +41 -0
- package/src/core/load.js +145 -0
- package/src/core/mask.js +207 -0
- package/src/core/pack.js +95 -0
- package/src/core/packlist.js +186 -0
- package/src/core/peer.js +39 -0
- package/src/core/profile-scan.js +121 -0
- package/src/core/report.js +174 -0
- package/src/core/rules.js +581 -0
- package/src/core/semver.js +192 -0
- package/src/core/tarball.js +90 -0
- package/src/data/event-types.json +393 -0
- package/src/experimental/conflicts.js +315 -0
- package/src/experimental/index.js +4 -0
- package/src/experimental/performance.js +515 -0
- package/src/experimental/security.js +462 -0
- package/src/index.js +74 -0
- package/src/load.js +1 -0
- package/src/mask.js +1 -0
- package/src/matrix/cache.js +19 -0
- package/src/matrix/cell.js +79 -0
- package/src/matrix/cells.js +58 -0
- package/src/matrix/conflicts.js +210 -0
- package/src/matrix/export.js +155 -0
- package/src/matrix/index.js +71 -0
- package/src/matrix/manifest.js +17 -0
- package/src/matrix/migrate.js +41 -0
- package/src/matrix/run.js +241 -0
- package/src/matrix/sanitize.js +42 -0
- package/src/matrix/schema.js +178 -0
- package/src/matrix/site.js +61 -0
- package/src/matrix/sync.js +72 -0
- package/src/matrix/verify.js +152 -0
- package/src/performance.js +1 -0
- package/src/probe/artifact-resolver.js +161 -0
- package/src/probe/classify.js +8 -0
- package/src/probe/dsh-adapter.js +212 -0
- package/src/probe/environment/docker.js +270 -0
- package/src/probe/environment/host-unsafe.js +39 -0
- package/src/probe/environment/materialize.js +22 -0
- package/src/probe/environment.js +50 -0
- package/src/probe/installed-artifact.js +17 -0
- package/src/probe/probe.js +278 -0
- package/src/report.js +1 -0
- package/src/rules.js +1 -0
- package/src/security.js +1 -0
- package/src/shared/errors.js +7 -0
- package/src/shared/hash.js +31 -0
- package/src/shared/status.js +12 -0
- package/src/shared/validate.js +14 -0
- package/src/shared/version.js +4 -0
- package/src/upgrade/compatibility.js +14 -0
- package/src/upgrade/planner.js +119 -0
- package/src/upgrade/profile.js +82 -0
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, readdirSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
import { ToolError } from '../shared/errors.js';
|
|
4
|
+
import { readCellAny } from './migrate.js';
|
|
5
|
+
import { validateCell } from './schema.js';
|
|
6
|
+
|
|
7
|
+
function segment(value) {
|
|
8
|
+
return String(value).replace(/^@/, '').replace(/[^A-Za-z0-9._-]/g, '_');
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function cellFilePath(outDir, cell) {
|
|
12
|
+
return join(
|
|
13
|
+
outDir,
|
|
14
|
+
segment(cell.plugin.name),
|
|
15
|
+
segment(cell.plugin.resolvedVersion),
|
|
16
|
+
`${segment(cell.dsh.version)}+${segment(cell.dsh.composition)}.${cell.cacheKey.slice(0, 16)}.json`,
|
|
17
|
+
);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function writeCell(outDir, cell) {
|
|
21
|
+
if (cell.schemaVersion !== 2) throw new ToolError('CELL_SCHEMA_UNSUPPORTED', 'Only v2 cells can be written by the matrix runner.');
|
|
22
|
+
if (cell.run.outcome === 'completed') {
|
|
23
|
+
const validation = validateCell(cell);
|
|
24
|
+
if (!validation.ok) throw new ToolError('CELL_INVALID', validation.errors.join('; '));
|
|
25
|
+
}
|
|
26
|
+
const path = cellFilePath(outDir, cell);
|
|
27
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
28
|
+
writeFileSync(path, JSON.stringify(cell, null, 2) + '\n');
|
|
29
|
+
return path;
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
export function readCells(dir) {
|
|
33
|
+
if (!existsSync(dir)) throw new ToolError('CELL_READ_FAILED', `The cells directory does not exist: ${dir}`);
|
|
34
|
+
const results = [];
|
|
35
|
+
function walk(current) {
|
|
36
|
+
for (const entry of readdirSync(current, { withFileTypes: true })) {
|
|
37
|
+
const path = join(current, entry.name);
|
|
38
|
+
if (entry.isDirectory()) {
|
|
39
|
+
walk(path);
|
|
40
|
+
continue;
|
|
41
|
+
}
|
|
42
|
+
if (!entry.isFile() || !entry.name.endsWith('.json')) continue;
|
|
43
|
+
try {
|
|
44
|
+
const value = JSON.parse(readFileSync(path, 'utf8'));
|
|
45
|
+
if (entry.name === 'run-manifest.json') {
|
|
46
|
+
results.push({ ok: true, path, kind: 'run-manifest', manifest: value });
|
|
47
|
+
continue;
|
|
48
|
+
}
|
|
49
|
+
const { kind, cell } = readCellAny(value);
|
|
50
|
+
results.push({ ok: true, path, kind, cell });
|
|
51
|
+
} catch (error) {
|
|
52
|
+
results.push({ ok: false, path, error: { code: error.code || 'CELL_INVALID', message: error.message } });
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
}
|
|
56
|
+
walk(dir);
|
|
57
|
+
return results.sort((a, b) => a.path.localeCompare(b.path));
|
|
58
|
+
}
|
|
@@ -0,0 +1,210 @@
|
|
|
1
|
+
import { execFile } from 'node:child_process';
|
|
2
|
+
import { existsSync, mkdirSync, mkdtempSync, readdirSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { join } from 'node:path';
|
|
4
|
+
import { promisify } from 'node:util';
|
|
5
|
+
import { detectConflicts } from '../experimental/conflicts.js';
|
|
6
|
+
import { loadPackage } from '../core/load.js';
|
|
7
|
+
import { ToolError } from '../shared/errors.js';
|
|
8
|
+
import { downloadArtifact, resolveArtifact } from '../probe/artifact-resolver.js';
|
|
9
|
+
import { createProbeEnvironment } from '../probe/environment.js';
|
|
10
|
+
import { resolveDsh, safeEnv } from '../probe/dsh-adapter.js';
|
|
11
|
+
import { readInstalledArtifact } from '../probe/installed-artifact.js';
|
|
12
|
+
import { classify } from '../probe/classify.js';
|
|
13
|
+
import { parseInactive, payloadOf } from '../probe/probe.js';
|
|
14
|
+
|
|
15
|
+
const execFileAsync = promisify(execFile);
|
|
16
|
+
|
|
17
|
+
function safeName(value) {
|
|
18
|
+
return String(value).replace(/^@/, '').replace(/[^A-Za-z0-9._-]/g, '_');
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function artifactFromCell(cell) {
|
|
22
|
+
return {
|
|
23
|
+
name: cell.plugin.name,
|
|
24
|
+
requestedVersion: cell.plugin.resolvedVersion,
|
|
25
|
+
resolvedVersion: cell.plugin.resolvedVersion,
|
|
26
|
+
integrity: cell.plugin.integrity,
|
|
27
|
+
sourceDigest: cell.plugin.sourceDigest,
|
|
28
|
+
local: cell.plugin.local,
|
|
29
|
+
path: cell.plugin.path,
|
|
30
|
+
tarballUrl: cell.plugin.tarballUrl,
|
|
31
|
+
installTarget: `${cell.plugin.name}@${cell.plugin.resolvedVersion}`,
|
|
32
|
+
};
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
function uniqueArtifacts(cells) {
|
|
36
|
+
const artifacts = new Map();
|
|
37
|
+
for (const cell of cells) {
|
|
38
|
+
const artifact = artifactFromCell(cell);
|
|
39
|
+
const key = `${artifact.name}@${artifact.resolvedVersion}@${artifact.integrity || artifact.sourceDigest}`;
|
|
40
|
+
if (!artifacts.has(key)) artifacts.set(key, { artifact, cells: [] });
|
|
41
|
+
artifacts.get(key).cells.push(cell);
|
|
42
|
+
}
|
|
43
|
+
return [...artifacts.values()];
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function assertSafeTarEntry(path) {
|
|
47
|
+
if (path.startsWith('/') || path === '..' || path.startsWith('../') || path.includes('/../') || path.includes('\0')) {
|
|
48
|
+
throw new ToolError('ARTIFACT_SPEC_UNSUPPORTED', 'The artifact contains an unsafe archive path.');
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
function assertNoSymlinks(dir) {
|
|
53
|
+
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
54
|
+
if (entry.isSymbolicLink()) throw new ToolError('ARTIFACT_SPEC_UNSUPPORTED', 'The artifact contains a symbolic link.');
|
|
55
|
+
if (entry.isDirectory()) assertNoSymlinks(join(dir, entry.name));
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function createPinnedMaterializer({ cacheRoot }) {
|
|
60
|
+
return async (artifact) => {
|
|
61
|
+
if (artifact.local && artifact.path) return artifact.path;
|
|
62
|
+
const pinned = await resolveArtifact({ name: artifact.name, version: artifact.resolvedVersion, integrity: artifact.integrity });
|
|
63
|
+
const downloaded = await downloadArtifact(pinned);
|
|
64
|
+
const destination = join(cacheRoot, 'pkgs', safeName(artifact.name), safeName(artifact.resolvedVersion));
|
|
65
|
+
if (existsSync(join(destination, 'package.json'))) return destination;
|
|
66
|
+
mkdirSync(cacheRoot, { recursive: true });
|
|
67
|
+
const listing = await execFileAsync('tar', ['-tzf', downloaded.tarballPath], { env: safeEnv(), maxBuffer: 16 * 1024 * 1024 });
|
|
68
|
+
for (const path of listing.stdout.split('\n').filter(Boolean)) assertSafeTarEntry(path);
|
|
69
|
+
const temporary = mkdtempSync(join(cacheRoot, 'extract-'));
|
|
70
|
+
try {
|
|
71
|
+
await execFileAsync('tar', ['-xzf', downloaded.tarballPath, '-C', temporary, '--strip-components=1', '--no-same-owner', '--no-same-permissions'], { env: safeEnv(), maxBuffer: 16 * 1024 * 1024 });
|
|
72
|
+
assertNoSymlinks(temporary);
|
|
73
|
+
if (!existsSync(join(temporary, 'package.json'))) throw new ToolError('ARTIFACT_MISSING', 'The extracted artifact has no package.json.');
|
|
74
|
+
rmSync(destination, { recursive: true, force: true });
|
|
75
|
+
mkdirSync(join(destination, '..'), { recursive: true });
|
|
76
|
+
renameSync(temporary, destination);
|
|
77
|
+
} catch (error) {
|
|
78
|
+
rmSync(temporary, { recursive: true, force: true });
|
|
79
|
+
throw error;
|
|
80
|
+
}
|
|
81
|
+
return destination;
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
|
|
85
|
+
async function verifyPinnedCombo({ artifacts, directories, dshVersion }) {
|
|
86
|
+
const environment = await createProbeEnvironment({ backend: 'auto' });
|
|
87
|
+
let stub;
|
|
88
|
+
try {
|
|
89
|
+
const dsh = await resolveDsh(dshVersion, environment.dshDir, { environment });
|
|
90
|
+
stub = await environment.startStub();
|
|
91
|
+
const patchPath = environment.isolation === 'docker' ? '/profile/probe-patch.yml' : join(environment.profileDir, 'probe-patch.yml');
|
|
92
|
+
writeFileSync(join(environment.profileDir, 'probe-patch.yml'), [
|
|
93
|
+
'- id: llm-deepseek',
|
|
94
|
+
' name: "@deepseek-ai/dsh-llm-deepseek-api-key"',
|
|
95
|
+
' config:',
|
|
96
|
+
` baseURL: ${stub.baseURL}`,
|
|
97
|
+
' apiKeyEnv: PROBE_API_KEY',
|
|
98
|
+
'',
|
|
99
|
+
].join('\n'));
|
|
100
|
+
const bootInput = {
|
|
101
|
+
dsh,
|
|
102
|
+
args: ['--profile', 'headless', '--patch', patchPath, 'probe'],
|
|
103
|
+
env: { PROBE_API_KEY: 'probe' },
|
|
104
|
+
network: stub.network,
|
|
105
|
+
};
|
|
106
|
+
await environment.boot(bootInput);
|
|
107
|
+
await stub.sync();
|
|
108
|
+
const before = payloadOf(stub.seen.splice(0));
|
|
109
|
+
for (const artifact of artifacts) {
|
|
110
|
+
const directory = directories.get(artifact.name);
|
|
111
|
+
const materialized = await environment.materialize({ ...artifact, local: true, path: directory });
|
|
112
|
+
await environment.install({ dsh, target: materialized, profile: 'headless', env: { PROBE_API_KEY: 'probe' } });
|
|
113
|
+
const installed = readInstalledArtifact(join(environment.profileDir, 'profiles', 'headless'), artifact.name);
|
|
114
|
+
if (installed.version !== artifact.resolvedVersion) throw new ToolError('VERSION_MISMATCH', 'A combination installed a different artifact version.');
|
|
115
|
+
}
|
|
116
|
+
const booted = await environment.boot(bootInput);
|
|
117
|
+
await stub.sync();
|
|
118
|
+
const after = payloadOf(stub.seen.splice(0));
|
|
119
|
+
const entries = parseInactive(booted.out);
|
|
120
|
+
const beforeNames = new Set(before?.tools?.map((tool) => tool.name) || []);
|
|
121
|
+
const afterNames = new Set(after?.tools?.map((tool) => tool.name) || []);
|
|
122
|
+
return {
|
|
123
|
+
status: classify(booted.out, entries, Boolean(after)),
|
|
124
|
+
trust: 'self-reported',
|
|
125
|
+
entries,
|
|
126
|
+
dropped: [...beforeNames].filter((name) => !afterNames.has(name)),
|
|
127
|
+
beforeCount: beforeNames.size,
|
|
128
|
+
afterCount: afterNames.size,
|
|
129
|
+
};
|
|
130
|
+
} finally {
|
|
131
|
+
if (stub?.server) stub.server.close();
|
|
132
|
+
await environment.destroy();
|
|
133
|
+
}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function candidateTools(cells) {
|
|
137
|
+
const owners = new Map();
|
|
138
|
+
for (const cell of cells) {
|
|
139
|
+
for (const tool of cell.result.tools?.added || []) {
|
|
140
|
+
if (!owners.has(tool)) owners.set(tool, new Map());
|
|
141
|
+
owners.get(tool).set(`${cell.plugin.name}@${cell.plugin.resolvedVersion}`, cell);
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
return [...owners.entries()].filter(([, byPlugin]) => byPlugin.size > 1).map(([resource, byPlugin]) => {
|
|
145
|
+
const claims = [...byPlugin.values()];
|
|
146
|
+
return {
|
|
147
|
+
resource,
|
|
148
|
+
plugins: claims.map((cell) => cell.plugin.name),
|
|
149
|
+
cells: claims,
|
|
150
|
+
status: 'candidate',
|
|
151
|
+
};
|
|
152
|
+
});
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
export async function scanConflicts({ cells, materialize, verify = false, verifyCombo = verifyPinnedCombo, limit = Number.POSITIVE_INFINITY }) {
|
|
156
|
+
const errors = [];
|
|
157
|
+
const unique = uniqueArtifacts(cells);
|
|
158
|
+
const packages = [];
|
|
159
|
+
const directories = new Map();
|
|
160
|
+
const pluginsScanned = [];
|
|
161
|
+
for (const entry of unique) {
|
|
162
|
+
try {
|
|
163
|
+
const directory = await materialize(entry.artifact);
|
|
164
|
+
directories.set(entry.artifact.name, directory);
|
|
165
|
+
packages.push(loadPackage(directory));
|
|
166
|
+
pluginsScanned.push(entry.artifact.name);
|
|
167
|
+
} catch (error) {
|
|
168
|
+
errors.push({ code: error.code || 'ARTIFACT_MATERIALIZE_FAILED', plugin: entry.artifact.name, message: error.message });
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
const staticReport = detectConflicts(packages);
|
|
172
|
+
const tools = candidateTools(cells);
|
|
173
|
+
const slots = staticReport.slots.map((entry) => ({ ...entry, status: 'candidate' }));
|
|
174
|
+
const routes = staticReport.routes.map((entry) => ({ ...entry, status: 'candidate' }));
|
|
175
|
+
const verified = { tools: [], slots: [], routes: [] };
|
|
176
|
+
if (verify) {
|
|
177
|
+
for (const [kind, candidates] of [['tools', tools], ['slots', slots], ['routes', routes]]) {
|
|
178
|
+
for (const candidate of candidates.slice(0, limit)) {
|
|
179
|
+
try {
|
|
180
|
+
const names = new Set(candidate.plugins);
|
|
181
|
+
const artifacts = unique.filter((entry) => names.has(entry.artifact.name)).map((entry) => entry.artifact);
|
|
182
|
+
const relatedCells = candidate.cells || unique.filter((entry) => names.has(entry.artifact.name)).flatMap((entry) => entry.cells);
|
|
183
|
+
const result = await verifyCombo({
|
|
184
|
+
kind,
|
|
185
|
+
candidate,
|
|
186
|
+
artifacts,
|
|
187
|
+
directories,
|
|
188
|
+
dshVersion: relatedCells[0]?.dsh.version,
|
|
189
|
+
composition: relatedCells[0]?.dsh.composition,
|
|
190
|
+
});
|
|
191
|
+
verified[kind].push({ resource: candidate.resource, plugins: candidate.plugins, ...result });
|
|
192
|
+
} catch (error) {
|
|
193
|
+
errors.push({ code: error.code || 'CONFLICT_VERIFICATION_FAILED', resource: candidate.resource, message: error.message });
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
return {
|
|
199
|
+
schemaVersion: 2,
|
|
200
|
+
generatedAt: new Date().toISOString(),
|
|
201
|
+
cells: cells.length,
|
|
202
|
+
pluginsScanned,
|
|
203
|
+
tools,
|
|
204
|
+
slots,
|
|
205
|
+
routes,
|
|
206
|
+
unknownRoutes: staticReport.unknownRoutes,
|
|
207
|
+
verified,
|
|
208
|
+
errors,
|
|
209
|
+
};
|
|
210
|
+
}
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, renameSync, rmSync, statSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { dirname, join, relative } from 'node:path';
|
|
3
|
+
import { ToolError } from '../shared/errors.js';
|
|
4
|
+
import { PROBE_VERSION } from '../shared/version.js';
|
|
5
|
+
import { stableStringify, sha256 } from '../shared/hash.js';
|
|
6
|
+
import { readCells, writeCell } from './cells.js';
|
|
7
|
+
import { buildIndex } from './index.js';
|
|
8
|
+
import { buildManifest } from './manifest.js';
|
|
9
|
+
import { validateCell, validateIndex, validateManifest } from './schema.js';
|
|
10
|
+
import { sanitizeCellForPublish, sanitizeForPublish } from './sanitize.js';
|
|
11
|
+
|
|
12
|
+
function readJson(path) {
|
|
13
|
+
return JSON.parse(readFileSync(path, 'utf8'));
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
function writeJson(path, value) {
|
|
17
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
18
|
+
writeFileSync(path, JSON.stringify(value, null, 2) + '\n');
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function filesUnder(dir) {
|
|
22
|
+
const files = [];
|
|
23
|
+
function walk(current) {
|
|
24
|
+
for (const entry of readdirSync(current, { withFileTypes: true })) {
|
|
25
|
+
const path = join(current, entry.name);
|
|
26
|
+
if (entry.isDirectory()) walk(path);
|
|
27
|
+
else if (entry.isFile()) files.push(relative(dir, path).replace(/\\/g, '/'));
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
walk(dir);
|
|
31
|
+
return files.sort();
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function buildAggregate(index, sourceRunId) {
|
|
35
|
+
const aggregate = {
|
|
36
|
+
schemaVersion: 2,
|
|
37
|
+
generatedAt: index.generatedAt,
|
|
38
|
+
sourceRunId,
|
|
39
|
+
dshVersions: index.dshVersions,
|
|
40
|
+
plugins: index.plugins.length,
|
|
41
|
+
versions: {},
|
|
42
|
+
};
|
|
43
|
+
for (const plugin of index.plugins) {
|
|
44
|
+
for (const version of plugin.versions) {
|
|
45
|
+
for (const cell of version.cells) {
|
|
46
|
+
if (!aggregate.versions[cell.dsh]) aggregate.versions[cell.dsh] = {};
|
|
47
|
+
aggregate.versions[cell.dsh][cell.status] = (aggregate.versions[cell.dsh][cell.status] || 0) + 1;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
}
|
|
51
|
+
return aggregate;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
export async function exportPublicArtifact({ cellsDir, conflictsPath, outDir, runId, generatedAt = new Date().toISOString(), expectedCells }) {
|
|
55
|
+
const reads = readCells(cellsDir);
|
|
56
|
+
const invalid = reads.filter((entry) => !entry.ok);
|
|
57
|
+
if (invalid.length > 0) throw new ToolError('EXPORT_INPUT_INVALID', `Cannot read ${invalid.length} cell file(s).`);
|
|
58
|
+
if (reads.some((entry) => entry.kind === 'legacy-v1')) throw new ToolError('EXPORT_INPUT_INVALID', 'Legacy cells cannot be republished as v2 evidence.');
|
|
59
|
+
const cells = reads.filter((entry) => entry.kind === 'v2').map((entry) => sanitizeCellForPublish(entry.cell));
|
|
60
|
+
if (expectedCells !== undefined && cells.length !== expectedCells) {
|
|
61
|
+
throw new ToolError('EXPORT_CELL_COUNT_MISMATCH', `Expected ${expectedCells} cells, found ${cells.length}.`);
|
|
62
|
+
}
|
|
63
|
+
if (!runId) throw new ToolError('EXPORT_INPUT_INVALID', 'runId is required.');
|
|
64
|
+
if (!existsSync(conflictsPath)) throw new ToolError('EXPORT_INPUT_INVALID', 'The computed conflicts artifact is required.');
|
|
65
|
+
const conflicts = sanitizeForPublish(readJson(conflictsPath));
|
|
66
|
+
if (conflicts.schemaVersion !== 2) throw new ToolError('EXPORT_INPUT_INVALID', 'Conflicts must use schema v2.');
|
|
67
|
+
const index = buildIndex(cells, { generatedAt, source: { runId, conflicts } });
|
|
68
|
+
const aggregate = buildAggregate(index, runId);
|
|
69
|
+
const sourceManifest = reads.find((entry) => entry.kind === 'run-manifest')?.manifest;
|
|
70
|
+
const parent = dirname(outDir);
|
|
71
|
+
mkdirSync(parent, { recursive: true });
|
|
72
|
+
const staging = mkdtempSync(join(parent, '.export-'));
|
|
73
|
+
try {
|
|
74
|
+
for (const cell of cells) writeCell(join(staging, 'cells'), cell);
|
|
75
|
+
writeJson(join(staging, 'index', 'latest.json'), index);
|
|
76
|
+
writeJson(join(staging, 'index', 'conflicts.json'), conflicts);
|
|
77
|
+
writeJson(join(staging, 'aggregate.json'), aggregate);
|
|
78
|
+
const artifacts = filesUnder(staging).map((path) => {
|
|
79
|
+
const bytes = readFileSync(join(staging, path));
|
|
80
|
+
return { path, bytes: bytes.length, sha256: sha256(bytes) };
|
|
81
|
+
});
|
|
82
|
+
const manifest = buildManifest({
|
|
83
|
+
runId,
|
|
84
|
+
generatedAt,
|
|
85
|
+
probeVersion: PROBE_VERSION,
|
|
86
|
+
inputs: { targetMode: sourceManifest?.targetMode || 'pinned', expectedCells: expectedCells ?? cells.length, inputs: sourceManifest?.inputs || index.plugins.map((plugin) => plugin.name) },
|
|
87
|
+
artifacts,
|
|
88
|
+
});
|
|
89
|
+
manifest.targetMode = sourceManifest?.targetMode || 'pinned';
|
|
90
|
+
manifest.inputs = sourceManifest?.inputs || index.plugins.map((plugin) => plugin.name);
|
|
91
|
+
manifest.cellCount = cells.length;
|
|
92
|
+
if (expectedCells !== undefined) manifest.expectedCells = expectedCells;
|
|
93
|
+
writeJson(join(staging, 'manifest.json'), manifest);
|
|
94
|
+
const verification = await verifyPublicArtifact({ dir: staging });
|
|
95
|
+
if (!verification.ok) throw new ToolError('EXPORT_INPUT_INVALID', verification.errors.join('; '));
|
|
96
|
+
const backup = `${outDir}.backup-${Date.now()}`;
|
|
97
|
+
if (existsSync(outDir)) renameSync(outDir, backup);
|
|
98
|
+
try {
|
|
99
|
+
renameSync(staging, outDir);
|
|
100
|
+
rmSync(backup, { recursive: true, force: true });
|
|
101
|
+
} catch (error) {
|
|
102
|
+
if (existsSync(backup)) renameSync(backup, outDir);
|
|
103
|
+
throw error;
|
|
104
|
+
}
|
|
105
|
+
return { manifest, filesWritten: artifacts.length + 1, cells: cells.length };
|
|
106
|
+
} catch (error) {
|
|
107
|
+
rmSync(staging, { recursive: true, force: true });
|
|
108
|
+
throw error;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
export async function verifyPublicArtifact({ dir }) {
|
|
113
|
+
const errors = [];
|
|
114
|
+
try {
|
|
115
|
+
const manifest = readJson(join(dir, 'manifest.json'));
|
|
116
|
+
const manifestValidation = validateManifest(manifest);
|
|
117
|
+
if (!manifestValidation.ok) errors.push(...manifestValidation.errors);
|
|
118
|
+
const declared = new Set(manifest.artifacts.map((artifact) => artifact.path));
|
|
119
|
+
const actual = new Set(filesUnder(dir).filter((path) => path !== 'manifest.json'));
|
|
120
|
+
for (const path of declared) {
|
|
121
|
+
if (!actual.has(path)) {
|
|
122
|
+
errors.push(`artifact is missing: ${path}`);
|
|
123
|
+
continue;
|
|
124
|
+
}
|
|
125
|
+
const expected = manifest.artifacts.find((artifact) => artifact.path === path);
|
|
126
|
+
const bytes = readFileSync(join(dir, path));
|
|
127
|
+
if (bytes.length !== expected.bytes || sha256(bytes) !== expected.sha256) errors.push(`artifact hash mismatch: ${path}`);
|
|
128
|
+
}
|
|
129
|
+
for (const path of actual) if (!declared.has(path)) errors.push(`undeclared artifact: ${path}`);
|
|
130
|
+
|
|
131
|
+
const index = readJson(join(dir, 'index', 'latest.json'));
|
|
132
|
+
const indexValidation = validateIndex(index);
|
|
133
|
+
if (!indexValidation.ok) errors.push(...indexValidation.errors);
|
|
134
|
+
const conflicts = readJson(join(dir, 'index', 'conflicts.json'));
|
|
135
|
+
if (conflicts.schemaVersion !== 2) errors.push('conflicts.schemaVersion must be 2');
|
|
136
|
+
const reads = readCells(join(dir, 'cells'));
|
|
137
|
+
const invalid = reads.filter((entry) => !entry.ok);
|
|
138
|
+
if (invalid.length > 0) errors.push(`${invalid.length} damaged public cell(s)`);
|
|
139
|
+
const cells = reads.filter((entry) => entry.kind === 'v2').map((entry) => entry.cell);
|
|
140
|
+
for (const cell of cells) {
|
|
141
|
+
const validation = validateCell(cell);
|
|
142
|
+
if (!validation.ok) errors.push(...validation.errors);
|
|
143
|
+
}
|
|
144
|
+
if (manifest.cellCount !== cells.length) errors.push(`cell count mismatch: manifest ${manifest.cellCount}, actual ${cells.length}`);
|
|
145
|
+
if (manifest.expectedCells !== undefined && manifest.expectedCells !== cells.length) errors.push(`expected cell count mismatch: ${manifest.expectedCells} vs ${cells.length}`);
|
|
146
|
+
const expectedIndex = buildIndex(cells, { generatedAt: index.generatedAt, source: index.source });
|
|
147
|
+
if (stableStringify(expectedIndex) !== stableStringify(index)) errors.push('index does not match the public cells');
|
|
148
|
+
const expectedAggregate = buildAggregate(index, manifest.runId);
|
|
149
|
+
const aggregate = readJson(join(dir, 'aggregate.json'));
|
|
150
|
+
if (stableStringify(expectedAggregate) !== stableStringify(aggregate)) errors.push('aggregate does not match the index');
|
|
151
|
+
} catch (error) {
|
|
152
|
+
errors.push(error.message);
|
|
153
|
+
}
|
|
154
|
+
return { ok: errors.length === 0, errors };
|
|
155
|
+
}
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { ToolError } from '../shared/errors.js';
|
|
2
|
+
import { validateCell, validateIndex } from './schema.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* JSON has no `undefined`. A cell that was probed without the tool inventory or
|
|
6
|
+
* without a system prompt would otherwise put `undefined` into the index, and
|
|
7
|
+
* the canonical hash (stableStringify) throws on it — so the artifact could be
|
|
8
|
+
* written but never verified. Drop those keys instead of inventing a value.
|
|
9
|
+
*/
|
|
10
|
+
function compact(value) {
|
|
11
|
+
if (Array.isArray(value)) return value.map(compact);
|
|
12
|
+
if (value && typeof value === 'object') {
|
|
13
|
+
const out = {};
|
|
14
|
+
for (const [key, entry] of Object.entries(value)) {
|
|
15
|
+
if (entry !== undefined) out[key] = compact(entry);
|
|
16
|
+
}
|
|
17
|
+
return out;
|
|
18
|
+
}
|
|
19
|
+
return value;
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export function buildIndex(cells, { generatedAt = new Date().toISOString(), source = {} } = {}) {
|
|
23
|
+
const grouped = new Map();
|
|
24
|
+
const dshVersions = new Set();
|
|
25
|
+
for (const cell of cells) {
|
|
26
|
+
if (cell.run?.outcome !== 'completed' || !validateCell(cell).ok) {
|
|
27
|
+
throw new ToolError('INDEX_INPUT_INVALID', 'Every input cell must be a complete, valid v2 observation.');
|
|
28
|
+
}
|
|
29
|
+
const name = cell.plugin.name;
|
|
30
|
+
const version = cell.plugin.resolvedVersion;
|
|
31
|
+
if (!grouped.has(name)) grouped.set(name, new Map());
|
|
32
|
+
const versions = grouped.get(name);
|
|
33
|
+
if (!versions.has(version)) versions.set(version, []);
|
|
34
|
+
versions.get(version).push(cell);
|
|
35
|
+
dshVersions.add(cell.dsh.version);
|
|
36
|
+
}
|
|
37
|
+
const plugins = [...grouped.entries()].sort(([a], [b]) => a.localeCompare(b)).map(([name, versions]) => ({
|
|
38
|
+
name,
|
|
39
|
+
versions: [...versions.entries()].sort(([a], [b]) => a.localeCompare(b, undefined, { numeric: true })).map(([version, entries]) => ({
|
|
40
|
+
version,
|
|
41
|
+
integrity: entries[0].plugin.integrity,
|
|
42
|
+
cells: entries.map((cell) => ({
|
|
43
|
+
cacheKey: cell.cacheKey,
|
|
44
|
+
dsh: cell.dsh.version,
|
|
45
|
+
composition: cell.dsh.composition,
|
|
46
|
+
status: cell.result.status,
|
|
47
|
+
trust: cell.run.trust,
|
|
48
|
+
at: cell.run.at,
|
|
49
|
+
note: cell.evidence?.note,
|
|
50
|
+
awaitedServices: cell.result.awaitedServices,
|
|
51
|
+
toolsAdded: cell.result.tools?.added || [],
|
|
52
|
+
audit: cell.result.audit,
|
|
53
|
+
systemPromptChars: cell.result.systemPrompt?.chars,
|
|
54
|
+
clientHalf: cell.result.clientHalf,
|
|
55
|
+
reproduce: cell.reproduce.command,
|
|
56
|
+
})),
|
|
57
|
+
})),
|
|
58
|
+
}));
|
|
59
|
+
const index = {
|
|
60
|
+
schemaVersion: 2,
|
|
61
|
+
generatedAt,
|
|
62
|
+
source,
|
|
63
|
+
dshVersions: [...dshVersions].sort((a, b) => a.localeCompare(b, undefined, { numeric: true })),
|
|
64
|
+
plugins,
|
|
65
|
+
conflicts: source.conflicts || { tools: [], slots: [], routes: [] },
|
|
66
|
+
};
|
|
67
|
+
const clean = compact(index);
|
|
68
|
+
const validation = validateIndex(clean);
|
|
69
|
+
if (!validation.ok) throw new ToolError('INDEX_INPUT_INVALID', validation.errors.join('; '));
|
|
70
|
+
return clean;
|
|
71
|
+
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { stableStringify, sha256 } from '../shared/hash.js';
|
|
2
|
+
import { ToolError } from '../shared/errors.js';
|
|
3
|
+
import { validateManifest } from './schema.js';
|
|
4
|
+
|
|
5
|
+
export function buildManifest({ runId, generatedAt = new Date().toISOString(), probeVersion, inputs, artifacts }) {
|
|
6
|
+
const manifest = {
|
|
7
|
+
schemaVersion: 2,
|
|
8
|
+
runId,
|
|
9
|
+
generatedAt,
|
|
10
|
+
probeVersion,
|
|
11
|
+
inputsSha256: sha256(stableStringify(inputs)),
|
|
12
|
+
artifacts: [...artifacts].sort((a, b) => a.path.localeCompare(b.path)),
|
|
13
|
+
};
|
|
14
|
+
const validation = validateManifest(manifest);
|
|
15
|
+
if (!validation.ok) throw new ToolError('MANIFEST_INVALID', validation.errors.join('; '));
|
|
16
|
+
return manifest;
|
|
17
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { ToolError } from '../shared/errors.js';
|
|
2
|
+
import { TRUST } from '../shared/status.js';
|
|
3
|
+
import { validateCell } from './schema.js';
|
|
4
|
+
|
|
5
|
+
export function migrateCellV1(value) {
|
|
6
|
+
if (!value || value.schemaVersion !== 1 || typeof value.plugin?.name !== 'string' || typeof value.plugin?.version !== 'string' || typeof value.dsh?.version !== 'string' || typeof value.result?.status !== 'string') {
|
|
7
|
+
throw new ToolError('CELL_INVALID', 'The legacy cell is missing required v1 fields.');
|
|
8
|
+
}
|
|
9
|
+
const cell = structuredClone(value);
|
|
10
|
+
const originalTrust = cell.run?.trust;
|
|
11
|
+
cell.run = { ...(cell.run || {}), outcome: cell.run?.outcome || 'completed', trust: TRUST.LEGACY_UNPINNED };
|
|
12
|
+
cell.plugin = {
|
|
13
|
+
...cell.plugin,
|
|
14
|
+
requestedVersion: cell.plugin.requestedVersion || 'legacy',
|
|
15
|
+
resolvedVersion: cell.plugin.version,
|
|
16
|
+
};
|
|
17
|
+
delete cell.plugin.loadedVersion;
|
|
18
|
+
cell.dsh = {
|
|
19
|
+
...cell.dsh,
|
|
20
|
+
composition: cell.dsh.composition || cell.result.composition || cell.dsh.profile || cell.probe?.mode || 'unknown',
|
|
21
|
+
};
|
|
22
|
+
cell.probe = {
|
|
23
|
+
...(cell.probe || {}),
|
|
24
|
+
version: cell.probe?.version || 'legacy',
|
|
25
|
+
isolation: cell.probe?.isolation || 'unknown',
|
|
26
|
+
host: cell.probe?.host || { os: cell.run.os || 'unknown', node: cell.run.node || 'unknown' },
|
|
27
|
+
};
|
|
28
|
+
cell.recommendationEligible = false;
|
|
29
|
+
cell.migration = { originalTrust };
|
|
30
|
+
return cell;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function readCellAny(value) {
|
|
34
|
+
if (value?.schemaVersion === 2) {
|
|
35
|
+
const validation = validateCell(value);
|
|
36
|
+
if (!validation.ok) throw new ToolError('CELL_INVALID', validation.errors.join('; '));
|
|
37
|
+
return { kind: 'v2', cell: value };
|
|
38
|
+
}
|
|
39
|
+
if (value?.schemaVersion === 1) return { kind: 'legacy-v1', cell: migrateCellV1(value) };
|
|
40
|
+
throw new ToolError('CELL_SCHEMA_UNSUPPORTED', 'Only matrix cell schema v1 and v2 are supported.');
|
|
41
|
+
}
|