@ngockhoale/ukit 3.4.74 → 3.4.75
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +4 -0
- package/manifests/platform.full.yaml +15 -0
- package/package.json +4 -1
- package/template_project/.claude/hooks/completion-gate.sh +10 -1
- package/template_project/.claude/hooks/handoff-resume.sh +9 -0
- package/template_project/.claude/hooks/owner-stop-gate.mjs +41 -0
- package/template_project/.claude/hooks/skill-router.sh +177 -16
- package/template_project/.claude/settings.json +11 -1
- package/template_project/.claude/ukit/index/publication-intent.mjs +122 -12
- package/template_project/.claude/ukit/index/route-task.mjs +47 -8
- package/template_project/.claude/ukit/runtime/cron-operation-adapter.mjs +11 -4
- package/template_project/.claude/ukit/runtime/execution-ledger.mjs +369 -29
- package/template_project/.claude/ukit/runtime/operation-evidence.mjs +9 -0
- package/template_project/.claude/ukit/runtime/resumable-run.mjs +33 -11
- package/template_project/.claude/ukit/runtime/stop-coordinator.mjs +158 -32
- package/template_project/.omp/hooks/pre/ukit-bridge.js +138 -3
- package/scripts/dev/worktree-preflight.mjs +0 -166
- package/scripts/measure-decision-gateway.mjs +0 -306
- package/scripts/probe/codex-capability-probe.mjs +0 -169
|
@@ -1,166 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
// Bootstrap the gitignored UKit runtime in a fresh `git worktree` so targeted tests can run.
|
|
3
|
-
//
|
|
4
|
-
// A new worktree only has tracked files. `.claude/`, `.ukit/`, `CLAUDE.md`, … are generated by
|
|
5
|
-
// `ukit install` and gitignored, so mirror/runtime tests fail there for environment reasons.
|
|
6
|
-
// This reuses `bin/ukit install` (no new generator), idempotently:
|
|
7
|
-
// - already bootstrapped → no-op
|
|
8
|
-
// - missing prerequisites → run `ukit install` once, re-check
|
|
9
|
-
// - not a linked worktree → refuse (the primary checkout's .claude/ is the maintainer runtime)
|
|
10
|
-
// Install can also create template-owned scaffolds missing from the checkout (docs/AI_HANDOFF/PLAN.md, SPEC.md);
|
|
11
|
-
// those are not gitignored, so they are reported as `visibleWrites` + a warning, never hidden or deleted.
|
|
12
|
-
// `ukit install` writes the install flavour of root CLAUDE.md/AGENTS.md, but those files are
|
|
13
|
-
// gitignored *render outputs* of template_project/instructions (docs:render). The preflight
|
|
14
|
-
// re-renders them when they drift — never a tracked render target, which is only reported.
|
|
15
|
-
// It copies nothing from another checkout (no sessions, secrets, handoff state) and does not
|
|
16
|
-
// link node_modules — run `yarn install` in the worktree (a symlinked node_modules is warned:
|
|
17
|
-
// realpath-sensitive hooks resolve it outside the project and misjudge delete targets).
|
|
18
|
-
//
|
|
19
|
-
// Usage: node scripts/dev/worktree-preflight.mjs [--root <dir>] [--check] [--json]
|
|
20
|
-
// Exit: 0 ready · 1 prerequisites missing (--check, or bootstrap failed) · 2 refused
|
|
21
|
-
|
|
22
|
-
import { spawnSync } from 'node:child_process';
|
|
23
|
-
import fs from 'node:fs';
|
|
24
|
-
import path from 'node:path';
|
|
25
|
-
import { fileURLToPath } from 'node:url';
|
|
26
|
-
|
|
27
|
-
// Generated by `ukit install`; each is required by a mirror/runtime test family.
|
|
28
|
-
export const PREREQUISITES = [
|
|
29
|
-
'node_modules',
|
|
30
|
-
'.claude/settings.json',
|
|
31
|
-
'.claude/hooks',
|
|
32
|
-
'.claude/ukit/runtime/execution-ledger.mjs',
|
|
33
|
-
'.ukit/storage/config.json',
|
|
34
|
-
'CLAUDE.md',
|
|
35
|
-
'AGENTS.md',
|
|
36
|
-
];
|
|
37
|
-
|
|
38
|
-
// Explicit bounds so a wedged git/render child can never hang the preflight. Normal runs take
|
|
39
|
-
// milliseconds; these sit far above that yet below `ukit install`'s own 120s budget.
|
|
40
|
-
const GIT_TIMEOUT_MS = 30_000;
|
|
41
|
-
const RENDER_TIMEOUT_MS = 60_000;
|
|
42
|
-
|
|
43
|
-
function git(root, args) {
|
|
44
|
-
const res = spawnSync('git', args, { cwd: root, encoding: 'utf8', timeout: GIT_TIMEOUT_MS });
|
|
45
|
-
return res.status === 0 ? res.stdout.trim() : null;
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
export function isLinkedWorktree(root) {
|
|
49
|
-
const gitDir = git(root, ['rev-parse', '--path-format=absolute', '--git-dir']);
|
|
50
|
-
const common = git(root, ['rev-parse', '--path-format=absolute', '--git-common-dir']);
|
|
51
|
-
return Boolean(gitDir && common && path.resolve(gitDir) !== path.resolve(common));
|
|
52
|
-
}
|
|
53
|
-
|
|
54
|
-
function isTracked(root, rel) {
|
|
55
|
-
const res = spawnSync('git', ['ls-files', '--error-unmatch', '--', rel], { cwd: root, stdio: 'ignore', timeout: GIT_TIMEOUT_MS });
|
|
56
|
-
// A timed-out probe proves nothing: treat as tracked so the file is reported, never overwritten.
|
|
57
|
-
return res.status === 0 || Boolean(res.error);
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
const RENDER_CLI = 'scripts/docs/render-instructions.mjs';
|
|
61
|
-
|
|
62
|
-
// Visible (non-ignored) paths in `git status`. `ukit install` creates template-owned scaffolds that
|
|
63
|
-
// are missing from the checkout (docs/AI_HANDOFF/PLAN.md, SPEC.md — mergeStrategy skip, handoff-clear
|
|
64
|
-
// restores them to template, never deletes them). They are not gitignored, so a HEAD that lost them
|
|
65
|
-
// turns the bootstrap into a visible write. Report it truthfully instead of hiding or deleting it.
|
|
66
|
-
function visibleStatus(root) {
|
|
67
|
-
const res = spawnSync('git', ['status', '--porcelain', '--untracked-files=all'], { cwd: root, encoding: 'utf8', timeout: GIT_TIMEOUT_MS });
|
|
68
|
-
return res.status === 0 ? new Set(res.stdout.split('\n').filter(Boolean)) : null;
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
// → rel paths whose rendered bytes differ from disk (`DRIFT <target>` lines of --check).
|
|
72
|
-
export function renderDrift(root) {
|
|
73
|
-
const res = spawnSync(process.execPath, [path.join(root, RENDER_CLI), '--check'], { cwd: root, encoding: 'utf8', timeout: RENDER_TIMEOUT_MS });
|
|
74
|
-
// A killed check prints nothing, which would read as "no drift" — fail loudly instead.
|
|
75
|
-
if (res.error) throw new Error(`render check did not complete (${res.error.code || res.error.message})`);
|
|
76
|
-
return `${res.stdout || ''}`.split('\n').map((l) => l.match(/^DRIFT (.+)$/)?.[1]).filter(Boolean);
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
export function missingPrerequisites(root) {
|
|
80
|
-
return PREREQUISITES.filter((rel) => !fs.existsSync(path.join(root, rel)));
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
export function preflight({ root, check = false } = {}) {
|
|
84
|
-
const projectRoot = fs.realpathSync(root);
|
|
85
|
-
const result = { root: projectRoot, ok: false, bootstrapped: false, missing: [], rendered: [], visibleWrites: [], warnings: [], home: null, error: null };
|
|
86
|
-
|
|
87
|
-
if (!fs.existsSync(path.join(projectRoot, 'bin/ukit'))) {
|
|
88
|
-
return { ...result, exitCode: 2, error: `not a UKit checkout (no bin/ukit): ${projectRoot}` };
|
|
89
|
-
}
|
|
90
|
-
if (!isLinkedWorktree(projectRoot)) {
|
|
91
|
-
return { ...result, exitCode: 2, error: 'refusing: not a linked worktree (git worktree add …); the primary checkout keeps its own runtime' };
|
|
92
|
-
}
|
|
93
|
-
|
|
94
|
-
result.missing = missingPrerequisites(projectRoot);
|
|
95
|
-
if (result.missing.length > 0) {
|
|
96
|
-
if (check) return { ...result, exitCode: 1 };
|
|
97
|
-
if (result.missing.includes('node_modules')) {
|
|
98
|
-
return { ...result, exitCode: 1, error: 'node_modules missing — run `yarn install` in the worktree first (not linked from another checkout)' };
|
|
99
|
-
}
|
|
100
|
-
|
|
101
|
-
// `ukit install` also writes user-level state (~/.ukit playbooks). Point HOME inside the
|
|
102
|
-
// worktree's ignored .cache/ so a preflight never mutates the real home or leaks from it.
|
|
103
|
-
result.home = path.join(projectRoot, '.cache/worktree-preflight-home');
|
|
104
|
-
fs.mkdirSync(result.home, { recursive: true });
|
|
105
|
-
const visibleBefore = visibleStatus(projectRoot);
|
|
106
|
-
const install = spawnSync(process.execPath, [path.join(projectRoot, 'bin/ukit'), 'install'], {
|
|
107
|
-
cwd: projectRoot,
|
|
108
|
-
encoding: 'utf8',
|
|
109
|
-
env: { ...process.env, HOME: result.home, USERPROFILE: result.home },
|
|
110
|
-
timeout: 120_000,
|
|
111
|
-
});
|
|
112
|
-
result.bootstrapped = true;
|
|
113
|
-
const visibleAfter = visibleStatus(projectRoot);
|
|
114
|
-
if (visibleBefore && visibleAfter) {
|
|
115
|
-
result.visibleWrites = [...visibleAfter].filter((l) => !visibleBefore.has(l)).map((l) => l.slice(3));
|
|
116
|
-
if (result.visibleWrites.length > 0) {
|
|
117
|
-
result.warnings.push(`ukit install created non-ignored file(s) missing from this checkout: ${result.visibleWrites.join(', ')} — template-owned scaffolds; restore/commit them (not deleted)`);
|
|
118
|
-
}
|
|
119
|
-
}
|
|
120
|
-
result.missing = missingPrerequisites(projectRoot);
|
|
121
|
-
if (install.status !== 0 || result.missing.length > 0) {
|
|
122
|
-
const tail = `${install.stdout || ''}${install.stderr || ''}`.trim().split('\n').slice(-5).join('\n');
|
|
123
|
-
return { ...result, exitCode: 1, error: `ukit install did not produce all prerequisites (exit ${install.status})\n${tail}` };
|
|
124
|
-
}
|
|
125
|
-
}
|
|
126
|
-
|
|
127
|
-
if (fs.lstatSync(path.join(projectRoot, 'node_modules')).isSymbolicLink()) {
|
|
128
|
-
result.warnings.push('node_modules is a symlink — realpath-sensitive tests (block-dangerous) fail; run `yarn install` in this worktree for a real copy');
|
|
129
|
-
}
|
|
130
|
-
|
|
131
|
-
const drift = renderDrift(projectRoot);
|
|
132
|
-
if (drift.length > 0) {
|
|
133
|
-
if (check) return { ...result, missing: drift.map((t) => `render:${t}`), exitCode: 1 };
|
|
134
|
-
const tracked = drift.filter((t) => isTracked(projectRoot, t));
|
|
135
|
-
if (tracked.length > 0) {
|
|
136
|
-
return { ...result, exitCode: 1, error: `tracked render target(s) drifted, not overwriting: ${tracked.join(', ')} — commit or revert the source edit` };
|
|
137
|
-
}
|
|
138
|
-
const write = spawnSync(process.execPath, [path.join(projectRoot, RENDER_CLI), '--write'], { cwd: projectRoot, encoding: 'utf8', timeout: RENDER_TIMEOUT_MS });
|
|
139
|
-
const still = renderDrift(projectRoot);
|
|
140
|
-
if (write.status !== 0 || still.length > 0) {
|
|
141
|
-
return { ...result, exitCode: 1, error: `docs:render did not clear drift: ${still.join(', ') || (write.stderr || '').trim()}` };
|
|
142
|
-
}
|
|
143
|
-
result.rendered = drift;
|
|
144
|
-
}
|
|
145
|
-
return { ...result, ok: true, exitCode: 0 };
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
function main(argv) {
|
|
149
|
-
const args = { root: process.cwd(), check: false, json: false };
|
|
150
|
-
for (let i = 0; i < argv.length; i += 1) {
|
|
151
|
-
if (argv[i] === '--root') args.root = path.resolve(argv[++i] || '');
|
|
152
|
-
else if (argv[i] === '--check') args.check = true;
|
|
153
|
-
else if (argv[i] === '--json') args.json = true;
|
|
154
|
-
}
|
|
155
|
-
const { exitCode, ...result } = preflight(args);
|
|
156
|
-
if (args.json) {
|
|
157
|
-
process.stdout.write(`${JSON.stringify(result)}\n`);
|
|
158
|
-
} else if (result.ok) {
|
|
159
|
-
console.log(`[worktree-preflight] ready${result.bootstrapped ? ' (bootstrapped via ukit install)' : ' (already bootstrapped)'}: ${result.root}`);
|
|
160
|
-
} else {
|
|
161
|
-
console.error(`[worktree-preflight] ${result.error || `missing: ${result.missing.join(', ')}`}`);
|
|
162
|
-
}
|
|
163
|
-
process.exit(exitCode);
|
|
164
|
-
}
|
|
165
|
-
|
|
166
|
-
if (process.argv[1] && path.resolve(process.argv[1]) === fileURLToPath(import.meta.url)) main(process.argv.slice(2));
|
|
@@ -1,306 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
// TASK-004 — G3-FR06 decision-gateway cost/latency measurement probe.
|
|
3
|
-
//
|
|
4
|
-
// Probes the configured unic-decision gateway through the real client path
|
|
5
|
-
// (createDecisionClient + resolveGatewayBaseUrl/ApiKey env indirection —
|
|
6
|
-
// NO new credentials) and writes evidence to
|
|
7
|
-
// docs/AI_HANDOFF/inventory/g3-gateway-measurement.{json,md}
|
|
8
|
-
//
|
|
9
|
-
// Rules (PLAN §5 / TASK-004):
|
|
10
|
-
// - cost is provider-reported usage when present, else literal 'unknown';
|
|
11
|
-
// chars/4 estimates are forbidden.
|
|
12
|
-
// - unreachable gateway → availability:'unavailable' + errorClass + zero
|
|
13
|
-
// samples — a recorded outcome, NOT a probe failure.
|
|
14
|
-
// - partial samples → samples:<n> + partial:true; never padded.
|
|
15
|
-
//
|
|
16
|
-
// Usage: node scripts/measure-decision-gateway.mjs [--samples N] [--out dir]
|
|
17
|
-
// Exit codes: 0 = artifact written (any availability), 2 = artifact write failed.
|
|
18
|
-
|
|
19
|
-
import { execSync } from 'node:child_process';
|
|
20
|
-
import fs from 'node:fs';
|
|
21
|
-
import path from 'node:path';
|
|
22
|
-
import { performance } from 'node:perf_hooks';
|
|
23
|
-
import { fileURLToPath } from 'node:url';
|
|
24
|
-
|
|
25
|
-
import { createDecisionClient } from '../src/decision/client.js';
|
|
26
|
-
import {
|
|
27
|
-
resolveGatewayApiKey,
|
|
28
|
-
resolveGatewayBaseUrl,
|
|
29
|
-
} from '../src/core/gatewayProbe.js';
|
|
30
|
-
|
|
31
|
-
const REPO_ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
|
|
32
|
-
const DEFAULT_SAMPLES = 20;
|
|
33
|
-
const DEFAULT_OUT_DIR = path.join(REPO_ROOT, 'docs/AI_HANDOFF/inventory');
|
|
34
|
-
const PROBE_BATCH_ID = 'g3-gateway-measurement';
|
|
35
|
-
const PROBE_QUESTIONS = [
|
|
36
|
-
{
|
|
37
|
-
decisionKey: 'probe.intent',
|
|
38
|
-
kind: 'choice',
|
|
39
|
-
instruction: 'Pick the intent kind.',
|
|
40
|
-
candidates: ['fix', 'feature', 'docs'],
|
|
41
|
-
},
|
|
42
|
-
];
|
|
43
|
-
|
|
44
|
-
// ---------- pure helpers (unit-tested) ----------
|
|
45
|
-
|
|
46
|
-
/**
|
|
47
|
-
* Nearest-rank percentile over a numeric sample array.
|
|
48
|
-
* @param {number[]} samples latencies in ms (need not be sorted)
|
|
49
|
-
* @param {number} p percentile in (0, 100]
|
|
50
|
-
* @returns {number|null} null for empty input — never fabricated
|
|
51
|
-
*/
|
|
52
|
-
export function percentile(samples, p) {
|
|
53
|
-
const clean = (samples ?? []).filter((v) => Number.isFinite(v)).sort((a, b) => a - b);
|
|
54
|
-
if (clean.length === 0) return null;
|
|
55
|
-
const rank = Math.ceil((p / 100) * clean.length);
|
|
56
|
-
return clean[Math.min(Math.max(rank, 1), clean.length) - 1];
|
|
57
|
-
}
|
|
58
|
-
|
|
59
|
-
/**
|
|
60
|
-
* Extract provider-reported cost/usage from a raw chat-completions JSON body.
|
|
61
|
-
* Returns the provider's `usage` object verbatim when present, else 'unknown'.
|
|
62
|
-
* Never estimates.
|
|
63
|
-
*/
|
|
64
|
-
export function extractCost(parsedBody) {
|
|
65
|
-
const usage = parsedBody?.usage;
|
|
66
|
-
if (usage && typeof usage === 'object' && Object.keys(usage).length > 0) {
|
|
67
|
-
return usage;
|
|
68
|
-
}
|
|
69
|
-
return 'unknown';
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
/**
|
|
73
|
-
* Build the machine-readable measurement artifact.
|
|
74
|
-
* @param {{availability:'ok'|'unavailable', samples:number[], failures:number,
|
|
75
|
-
* attempted:number, checkpoint:string|null, cost:object|'unknown',
|
|
76
|
-
* errorClass?:string|null, endpointSource?:string|null}} probe
|
|
77
|
-
*/
|
|
78
|
-
export function buildMeasurementArtifact(probe, { gitSha, measuredAt } = {}) {
|
|
79
|
-
const latencies = probe?.samples ?? [];
|
|
80
|
-
const artifact = {
|
|
81
|
-
schema: 'g3-gateway-measurement/1',
|
|
82
|
-
availability: probe?.availability === 'ok' ? 'ok' : 'unavailable',
|
|
83
|
-
samples: latencies.length,
|
|
84
|
-
attempted: probe?.attempted ?? latencies.length,
|
|
85
|
-
failures: probe?.failures ?? 0,
|
|
86
|
-
p50Ms: percentile(latencies, 50),
|
|
87
|
-
p95Ms: percentile(latencies, 95),
|
|
88
|
-
minMs: latencies.length ? Math.min(...latencies) : null,
|
|
89
|
-
maxMs: latencies.length ? Math.max(...latencies) : null,
|
|
90
|
-
checkpoint: probe?.checkpoint ?? null,
|
|
91
|
-
cost: probe?.cost ?? 'unknown',
|
|
92
|
-
measuredAt: measuredAt ?? new Date().toISOString(),
|
|
93
|
-
gitSha: gitSha ?? null,
|
|
94
|
-
};
|
|
95
|
-
if (probe?.errorClass) artifact.errorClass = probe.errorClass;
|
|
96
|
-
if (probe?.endpointSource) artifact.endpointSource = probe.endpointSource;
|
|
97
|
-
// Honest sample count: anything below the attempted size is partial, never padded.
|
|
98
|
-
if (artifact.attempted > 0 && artifact.samples < artifact.attempted) {
|
|
99
|
-
artifact.partial = true;
|
|
100
|
-
}
|
|
101
|
-
return artifact;
|
|
102
|
-
}
|
|
103
|
-
|
|
104
|
-
/** Render the markdown evidence summary from an artifact object. */
|
|
105
|
-
export function renderMeasurementMarkdown(artifact) {
|
|
106
|
-
const lines = [
|
|
107
|
-
'# G3 Gateway Measurement — unic-decision',
|
|
108
|
-
'',
|
|
109
|
-
`- Measured at: ${artifact.measuredAt}`,
|
|
110
|
-
`- Git SHA: ${artifact.gitSha ?? 'unknown'}`,
|
|
111
|
-
`- Availability: **${artifact.availability}**`,
|
|
112
|
-
`- Checkpoint: ${artifact.checkpoint ?? 'unresolved'}`,
|
|
113
|
-
`- Endpoint source: ${artifact.endpointSource ?? 'none'}`,
|
|
114
|
-
`- Samples: ${artifact.samples} ok / ${artifact.attempted} attempted` +
|
|
115
|
-
(artifact.partial ? ' (**partial**)' : ''),
|
|
116
|
-
`- p50 latency: ${artifact.p50Ms ?? 'n/a'} ms`,
|
|
117
|
-
`- p95 latency: ${artifact.p95Ms ?? 'n/a'} ms`,
|
|
118
|
-
`- min/max: ${artifact.minMs ?? 'n/a'} / ${artifact.maxMs ?? 'n/a'} ms`,
|
|
119
|
-
`- Cost (provider-reported): ${artifact.cost === 'unknown' ? 'unknown' : JSON.stringify(artifact.cost)}`,
|
|
120
|
-
];
|
|
121
|
-
if (artifact.errorClass) lines.push(`- Error class: ${artifact.errorClass}`);
|
|
122
|
-
const interpretation =
|
|
123
|
-
artifact.availability === 'ok'
|
|
124
|
-
? `Gateway reachable; ${artifact.samples} successful decision-batch ` +
|
|
125
|
-
`probes measured over the real client path (createDecisionClient → ` +
|
|
126
|
-
`resolveGatewayBaseUrl/ApiKey). Owner classification: local Lava/JEV ` +
|
|
127
|
-
`deployment, near-zero marginal cost [owner-cited context, not measured].`
|
|
128
|
-
: artifact.errorClass === 'endpoint-rejected'
|
|
129
|
-
? 'Endpoint resolved and answered HTTP, but every decision-batch probe ' +
|
|
130
|
-
'was rejected (400/404/422 → `endpoint-rejected`) — the gateway is ' +
|
|
131
|
-
'reachable yet does not accept the C13 tools-encoded batch on this ' +
|
|
132
|
-
'checkpoint. No latency could be sampled; fallback path absorbs this.'
|
|
133
|
-
: 'Gateway was NOT reachable (or no endpoint configured) during this ' +
|
|
134
|
-
'probe. This is a recorded outcome — the scheduler fallback path ' +
|
|
135
|
-
'absorbs it; re-run the probe when the gateway is expected to be up.';
|
|
136
|
-
lines.push(
|
|
137
|
-
'',
|
|
138
|
-
'## Interpretation',
|
|
139
|
-
'',
|
|
140
|
-
interpretation,
|
|
141
|
-
'',
|
|
142
|
-
'## Explicit unknowns',
|
|
143
|
-
'',
|
|
144
|
-
artifact.cost === 'unknown'
|
|
145
|
-
? '- `cost`: the provider response carried no `usage` block → recorded `unknown` (estimates forbidden).'
|
|
146
|
-
: '- `cost`: recorded verbatim from the provider `usage` block.',
|
|
147
|
-
artifact.checkpoint === null
|
|
148
|
-
? '- `checkpoint`: unresolved (no successful request).'
|
|
149
|
-
: null,
|
|
150
|
-
'',
|
|
151
|
-
);
|
|
152
|
-
return lines.filter((l) => l !== null).join('\n');
|
|
153
|
-
}
|
|
154
|
-
|
|
155
|
-
export function writeMeasurementArtifacts(artifact, outDir) {
|
|
156
|
-
fs.mkdirSync(outDir, { recursive: true });
|
|
157
|
-
const jsonPath = path.join(outDir, 'g3-gateway-measurement.json');
|
|
158
|
-
const mdPath = path.join(outDir, 'g3-gateway-measurement.md');
|
|
159
|
-
fs.writeFileSync(jsonPath, `${JSON.stringify(artifact, null, 2)}\n`, 'utf8');
|
|
160
|
-
fs.writeFileSync(mdPath, renderMeasurementMarkdown(artifact), 'utf8');
|
|
161
|
-
return { jsonPath, mdPath };
|
|
162
|
-
}
|
|
163
|
-
|
|
164
|
-
// ---------- live probe ----------
|
|
165
|
-
|
|
166
|
-
function gitSha() {
|
|
167
|
-
try {
|
|
168
|
-
return execSync('git rev-parse HEAD', { cwd: REPO_ROOT, encoding: 'utf8', timeout: 10_000 }).trim();
|
|
169
|
-
} catch {
|
|
170
|
-
return null;
|
|
171
|
-
}
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
function parseArgs(argv) {
|
|
175
|
-
const opts = { samples: DEFAULT_SAMPLES, outDir: DEFAULT_OUT_DIR };
|
|
176
|
-
for (let i = 0; i < argv.length; i += 1) {
|
|
177
|
-
if (argv[i] === '--samples') {
|
|
178
|
-
i += 1;
|
|
179
|
-
opts.samples = Number(argv[i]);
|
|
180
|
-
} else if (argv[i] === '--out') {
|
|
181
|
-
i += 1;
|
|
182
|
-
opts.outDir = path.resolve(argv[i]);
|
|
183
|
-
} else if (argv[i] === '--help' || argv[i] === '-h') {
|
|
184
|
-
return { help: true };
|
|
185
|
-
} else {
|
|
186
|
-
throw new Error(`Unknown argument: ${argv[i]}`);
|
|
187
|
-
}
|
|
188
|
-
}
|
|
189
|
-
return opts;
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
/** Wrap fetch to capture the last successful response's provider `usage`. */
|
|
193
|
-
function captureCostTransport(baseTransport, sink) {
|
|
194
|
-
return async (url, init) => {
|
|
195
|
-
const res = await baseTransport(url, init);
|
|
196
|
-
if (res?.ok !== false && typeof res?.clone === 'function') {
|
|
197
|
-
try {
|
|
198
|
-
const body = await res.clone().json();
|
|
199
|
-
const cost = extractCost(body);
|
|
200
|
-
if (cost !== 'unknown') sink.cost = cost;
|
|
201
|
-
} catch {
|
|
202
|
-
// clone/parse failure must never break the probe
|
|
203
|
-
}
|
|
204
|
-
}
|
|
205
|
-
return res;
|
|
206
|
-
};
|
|
207
|
-
}
|
|
208
|
-
|
|
209
|
-
async function runProbe({ samples }) {
|
|
210
|
-
const projectRoot = REPO_ROOT;
|
|
211
|
-
const env = process.env;
|
|
212
|
-
const base = await resolveGatewayBaseUrl({ projectRoot, env });
|
|
213
|
-
const key = await resolveGatewayApiKey({ projectRoot, env });
|
|
214
|
-
const endpointSource = base?.source ?? null;
|
|
215
|
-
|
|
216
|
-
if (!base?.baseUrl) {
|
|
217
|
-
return {
|
|
218
|
-
availability: 'unavailable',
|
|
219
|
-
errorClass: 'endpoint-unconfigured',
|
|
220
|
-
endpointSource: null,
|
|
221
|
-
samples: [],
|
|
222
|
-
failures: 0,
|
|
223
|
-
attempted: 0,
|
|
224
|
-
checkpoint: null,
|
|
225
|
-
cost: 'unknown',
|
|
226
|
-
};
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
const sink = { cost: 'unknown' };
|
|
230
|
-
const client = createDecisionClient({
|
|
231
|
-
config: {},
|
|
232
|
-
transport: captureCostTransport(globalThis.fetch.bind(globalThis), sink),
|
|
233
|
-
projectRoot,
|
|
234
|
-
env,
|
|
235
|
-
});
|
|
236
|
-
|
|
237
|
-
const health = await client.healthProbe();
|
|
238
|
-
const latencies = [];
|
|
239
|
-
let failures = 0;
|
|
240
|
-
let lastErrorClass = null;
|
|
241
|
-
let checkpoint = health?.checkpoint ?? null;
|
|
242
|
-
|
|
243
|
-
if (health.status === 'ok' || health.status === 'unavailable' || health.status === 'timeout') {
|
|
244
|
-
for (let i = 0; i < samples; i += 1) {
|
|
245
|
-
const started = performance.now();
|
|
246
|
-
const result = await client.requestBatch({
|
|
247
|
-
batchId: `${PROBE_BATCH_ID}-${i}`,
|
|
248
|
-
boundary: 'measurement',
|
|
249
|
-
statePacket: { stateVersion: 1, taskClass: 'measurement', probeIndex: i },
|
|
250
|
-
questions: PROBE_QUESTIONS,
|
|
251
|
-
deadlineMs: 10_000,
|
|
252
|
-
});
|
|
253
|
-
const elapsed = performance.now() - started;
|
|
254
|
-
if (result.status === 'ok' || result.status === 'accepted' || result.answers?.length > 0) {
|
|
255
|
-
latencies.push(Math.round(elapsed * 100) / 100);
|
|
256
|
-
checkpoint = checkpoint ?? result.checkpoint ?? null;
|
|
257
|
-
} else {
|
|
258
|
-
failures += 1;
|
|
259
|
-
lastErrorClass = result.fallbackCode ?? result.status;
|
|
260
|
-
checkpoint = checkpoint ?? result.checkpoint ?? null;
|
|
261
|
-
}
|
|
262
|
-
}
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
const available = latencies.length > 0;
|
|
266
|
-
return {
|
|
267
|
-
availability: available ? 'ok' : 'unavailable',
|
|
268
|
-
errorClass: available
|
|
269
|
-
? null
|
|
270
|
-
: (lastErrorClass ?? (health.status === 'ok' ? null : health.status)),
|
|
271
|
-
endpointSource: base.source ?? (key?.source ? `key:${key.source}` : null),
|
|
272
|
-
samples: latencies,
|
|
273
|
-
failures,
|
|
274
|
-
attempted: samples,
|
|
275
|
-
checkpoint,
|
|
276
|
-
cost: sink.cost,
|
|
277
|
-
healthStatus: health.status,
|
|
278
|
-
};
|
|
279
|
-
}
|
|
280
|
-
|
|
281
|
-
const isMain = process.argv[1] && fileURLToPath(import.meta.url) === path.resolve(process.argv[1]);
|
|
282
|
-
if (isMain) {
|
|
283
|
-
try {
|
|
284
|
-
const opts = parseArgs(process.argv.slice(2));
|
|
285
|
-
if (opts.help) {
|
|
286
|
-
console.log('Usage: node scripts/measure-decision-gateway.mjs [--samples N] [--out dir]');
|
|
287
|
-
process.exit(0);
|
|
288
|
-
}
|
|
289
|
-
if (!Number.isInteger(opts.samples) || opts.samples < 1) {
|
|
290
|
-
throw new Error('--samples must be a positive integer');
|
|
291
|
-
}
|
|
292
|
-
const probe = await runProbe(opts);
|
|
293
|
-
const artifact = buildMeasurementArtifact(probe, {
|
|
294
|
-
gitSha: gitSha(),
|
|
295
|
-
measuredAt: new Date().toISOString(),
|
|
296
|
-
});
|
|
297
|
-
const { jsonPath, mdPath } = writeMeasurementArtifacts(artifact, opts.outDir);
|
|
298
|
-
console.log(`availability=${artifact.availability} samples=${artifact.samples} p50=${artifact.p50Ms} p95=${artifact.p95Ms} cost=${artifact.cost === 'unknown' ? 'unknown' : 'provider'}`);
|
|
299
|
-
console.log(`wrote ${jsonPath}`);
|
|
300
|
-
console.log(`wrote ${mdPath}`);
|
|
301
|
-
process.exit(0);
|
|
302
|
-
} catch (err) {
|
|
303
|
-
console.error(`measure-decision-gateway: ${err?.message ?? err}`);
|
|
304
|
-
process.exit(2);
|
|
305
|
-
}
|
|
306
|
-
}
|
|
@@ -1,169 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env node
|
|
2
|
-
// codex-capability-probe.mjs — TASK-001 / BL-001 + BL-002 scripted probe.
|
|
3
|
-
//
|
|
4
|
-
// Probes the installed `.codex/` surface of a UKit workspace and records what
|
|
5
|
-
// can be shown WITHOUT a live Codex CLI session:
|
|
6
|
-
// 1. codex CLI presence on PATH (live-session gate).
|
|
7
|
-
// 2. `.codex/` surface inventory (symlinks + files shipped by install).
|
|
8
|
-
// 3. `.codex/agents` resolution into real agent definitions.
|
|
9
|
-
// 4. Review-lane script reachability: `review-verdict.mjs` and
|
|
10
|
-
// `review-panel-aggregate.mjs` invoked THROUGH the `.codex/ukit` symlink
|
|
11
|
-
// (the exact path instruction-mediated Codex Bash would use).
|
|
12
|
-
// 5. Instruction mediation: `.codex/README.md` / `settings.json` point the
|
|
13
|
-
// model at `.codex/ukit/index/*` helpers.
|
|
14
|
-
//
|
|
15
|
-
// Honesty contract (SPEC §14): output states are PROVEN / ADVISORY-REACHABLE /
|
|
16
|
-
// INCONCLUSIVE / UNSUPPORTED — a cell can never be promoted beyond what the
|
|
17
|
-
// printed evidence shows. `codex-cli: absent` ⇒ subagent consumption is
|
|
18
|
-
// INCONCLUSIVE by construction; no claim of tested capability is emitted.
|
|
19
|
-
//
|
|
20
|
-
// Usage: node scripts/probe/codex-capability-probe.mjs [workspace-root]
|
|
21
|
-
|
|
22
|
-
import fs from 'node:fs';
|
|
23
|
-
import os from 'node:os';
|
|
24
|
-
import path from 'node:path';
|
|
25
|
-
import process from 'node:process';
|
|
26
|
-
import { spawnSync } from 'node:child_process';
|
|
27
|
-
|
|
28
|
-
// The probe targets an INSTALLED workspace (.codex/ exists after `ukit
|
|
29
|
-
// install`); pass the workspace root explicitly when it differs from cwd.
|
|
30
|
-
const ROOT = path.resolve(process.argv[2] ?? process.cwd());
|
|
31
|
-
const CODEX_DIR = path.join(ROOT, '.codex');
|
|
32
|
-
|
|
33
|
-
const out = [];
|
|
34
|
-
const say = (line = '') => { out.push(line); console.log(line); };
|
|
35
|
-
|
|
36
|
-
function commandOnPath(cmd) {
|
|
37
|
-
const r = spawnSync('sh', ['-c', `command -v ${cmd}`], { encoding: 'utf8', timeout: 5000 });
|
|
38
|
-
return r.status === 0 ? r.stdout.trim() : null;
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
function lstatKind(p) {
|
|
42
|
-
try {
|
|
43
|
-
const st = fs.lstatSync(p);
|
|
44
|
-
if (st.isSymbolicLink()) return `symlink -> ${fs.readlinkSync(p)}`;
|
|
45
|
-
if (st.isDirectory()) return 'dir';
|
|
46
|
-
return `file (${st.size}B)`;
|
|
47
|
-
} catch {
|
|
48
|
-
return 'ABSENT';
|
|
49
|
-
}
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
say(`probe-root: ${ROOT}`);
|
|
53
|
-
say(`probe-at: ${new Date().toISOString()}`);
|
|
54
|
-
|
|
55
|
-
// --- 1. codex CLI presence -------------------------------------------------
|
|
56
|
-
const codexBin = commandOnPath('codex');
|
|
57
|
-
say(`codex-cli: ${codexBin ? `present (${codexBin})` : 'absent'}`);
|
|
58
|
-
|
|
59
|
-
// --- 2. .codex surface inventory ------------------------------------------
|
|
60
|
-
say('--- .codex surface inventory ---');
|
|
61
|
-
let codexEntries = [];
|
|
62
|
-
try {
|
|
63
|
-
codexEntries = fs.readdirSync(CODEX_DIR).sort();
|
|
64
|
-
} catch {
|
|
65
|
-
say('.codex/: ABSENT (run `node bin/ukit install` first)');
|
|
66
|
-
}
|
|
67
|
-
for (const name of codexEntries) {
|
|
68
|
-
say(` .codex/${name}: ${lstatKind(path.join(CODEX_DIR, name))}`);
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
// --- 3. .codex/agents resolution -------------------------------------------
|
|
72
|
-
say('--- .codex/agents resolution ---');
|
|
73
|
-
const agentsDir = path.join(CODEX_DIR, 'agents');
|
|
74
|
-
let agentFiles = [];
|
|
75
|
-
try {
|
|
76
|
-
agentFiles = fs.readdirSync(agentsDir).filter((f) => f.endsWith('.md')).sort();
|
|
77
|
-
} catch {
|
|
78
|
-
// absent
|
|
79
|
-
}
|
|
80
|
-
say(` resolved-agent-files: ${agentFiles.length} [${agentFiles.join(', ')}]`);
|
|
81
|
-
for (const wanted of ['code-reviewer.md', 'handoff-planner.md', 'handoff-review.md']) {
|
|
82
|
-
say(` ${wanted}: ${agentFiles.includes(wanted) ? 'present' : 'absent'}`);
|
|
83
|
-
}
|
|
84
|
-
|
|
85
|
-
// --- 4. review-lane reachability via .codex/ukit ---------------------------
|
|
86
|
-
say('--- review-lane reachability (via .codex/ukit symlink) ---');
|
|
87
|
-
const verdictCli = path.join(CODEX_DIR, 'ukit', 'index', 'review-verdict.mjs');
|
|
88
|
-
const panelCli = path.join(CODEX_DIR, 'ukit', 'index', 'review-panel-aggregate.mjs');
|
|
89
|
-
say(` .codex/ukit/index/review-verdict.mjs: ${fs.existsSync(verdictCli) ? 'resolves' : 'missing'}`);
|
|
90
|
-
say(` .codex/ukit/index/review-panel-aggregate.mjs: ${fs.existsSync(panelCli) ? 'resolves' : 'missing'}`);
|
|
91
|
-
|
|
92
|
-
let verdictReachable = false;
|
|
93
|
-
let panelReachable = false;
|
|
94
|
-
if (fs.existsSync(verdictCli) || fs.existsSync(panelCli)) {
|
|
95
|
-
const tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'ukit-codex-probe-'));
|
|
96
|
-
try {
|
|
97
|
-
if (fs.existsSync(verdictCli)) {
|
|
98
|
-
const findings = path.join(tmp, 'findings.json');
|
|
99
|
-
fs.writeFileSync(findings, JSON.stringify({
|
|
100
|
-
findings: [{ id: 'P1', severity: 'minor', file: 'probe.js', line: 1, claim: 'probe finding' }],
|
|
101
|
-
}));
|
|
102
|
-
// Isolate config: empty root/home => decisionPlane stage resolves 'off',
|
|
103
|
-
// so no gateway call — deterministic fallback is the demonstrated path.
|
|
104
|
-
const r = spawnSync('node', [verdictCli, '--file', findings], {
|
|
105
|
-
encoding: 'utf8',
|
|
106
|
-
env: { ...process.env, UKIT_TEST_ROOT: tmp, UKIT_TEST_HOME: tmp },
|
|
107
|
-
timeout: 15000,
|
|
108
|
-
});
|
|
109
|
-
const parsed = JSON.parse(r.stdout.trim() || '{}');
|
|
110
|
-
say(` review-verdict run: exit=${r.status} verdict=${parsed.verdict ?? 'UNPARSED'} `
|
|
111
|
-
+ `stage=${parsed.stage ?? '-'} verdictSource=${parsed.verdictSource ?? '-'} `
|
|
112
|
-
+ `outcomeClass=${parsed.outcomeClass ?? '-'}`);
|
|
113
|
-
verdictReachable = r.status === 0 && typeof parsed.verdict === 'string';
|
|
114
|
-
}
|
|
115
|
-
if (fs.existsSync(panelCli)) {
|
|
116
|
-
const task = path.join(tmp, 'PROBE-TASK.md');
|
|
117
|
-
fs.writeFileSync(task, [
|
|
118
|
-
'# probe task',
|
|
119
|
-
'## Reviewer Verdict',
|
|
120
|
-
'MEMBER: probe-reviewer-a',
|
|
121
|
-
'VERDICT: APPROVED',
|
|
122
|
-
'## Reviewer Verdict',
|
|
123
|
-
'MEMBER: probe-reviewer-b',
|
|
124
|
-
'VERDICT: APPROVED',
|
|
125
|
-
'## Other',
|
|
126
|
-
'tail',
|
|
127
|
-
'',
|
|
128
|
-
].join('\n'));
|
|
129
|
-
const r = spawnSync('node', [panelCli, task], { encoding: 'utf8', timeout: 15000 });
|
|
130
|
-
const verdictLine = (r.stdout || '').split('\n').find((l) => /^VERDICT:/.test(l)) ?? '(no VERDICT line)';
|
|
131
|
-
say(` review-panel-aggregate run: exit=${r.status} ${verdictLine.trim()}`);
|
|
132
|
-
panelReachable = r.status === 0 && /^VERDICT: approved/.test(verdictLine.trim());
|
|
133
|
-
}
|
|
134
|
-
} finally {
|
|
135
|
-
fs.rmSync(tmp, { recursive: true, force: true });
|
|
136
|
-
}
|
|
137
|
-
}
|
|
138
|
-
|
|
139
|
-
// --- 5. instruction mediation ----------------------------------------------
|
|
140
|
-
say('--- instruction mediation ---');
|
|
141
|
-
const readmePath = path.join(CODEX_DIR, 'README.md');
|
|
142
|
-
const settingsPath = path.join(CODEX_DIR, 'settings.json');
|
|
143
|
-
try {
|
|
144
|
-
const readme = fs.readFileSync(readmePath, 'utf8');
|
|
145
|
-
const hits = (readme.match(/\.codex\/ukit\/index\/[\w-]+\.mjs/g) ?? []);
|
|
146
|
-
say(` .codex/README.md helper references: ${[...new Set(hits)].join(', ') || 'none'}`);
|
|
147
|
-
} catch {
|
|
148
|
-
say(' .codex/README.md: ABSENT');
|
|
149
|
-
}
|
|
150
|
-
try {
|
|
151
|
-
const settings = JSON.parse(fs.readFileSync(settingsPath, 'utf8'));
|
|
152
|
-
const helper = settings?.skillRouter?.helperCommand ?? '(none)';
|
|
153
|
-
say(` .codex/settings.json skillRouter.helperCommand: ${helper}`);
|
|
154
|
-
} catch {
|
|
155
|
-
say(' .codex/settings.json: ABSENT/unparseable');
|
|
156
|
-
}
|
|
157
|
-
|
|
158
|
-
// --- verdicts ---------------------------------------------------------------
|
|
159
|
-
say('--- verdicts ---');
|
|
160
|
-
const agentsSurface = agentFiles.length > 0;
|
|
161
|
-
say(`codex.subagents: ${codexBin ? 'PENDING-LIVE' : 'INCONCLUSIVE'}`
|
|
162
|
-
+ ` (.codex/agents surface ${agentsSurface ? 'shipped+resolved' : 'not resolvable'};`
|
|
163
|
-
+ ` host consumption ${codexBin ? 'requires live-session drive' : 'unprovable — codex CLI absent'})`);
|
|
164
|
-
say(`codex.review-lane: ${verdictReachable && panelReachable ? 'ADVISORY-REACHABLE' : 'INCONCLUSIVE'}`
|
|
165
|
-
+ ` (instruction-mediated Bash via .codex/ukit symlink;`
|
|
166
|
-
+ ` review-verdict=${verdictReachable ? 'ran' : 'failed'},`
|
|
167
|
-
+ ` panel-aggregate=${panelReachable ? 'ran' : 'failed'};`
|
|
168
|
-
+ ' enforcement: advisory — Codex exposes no hook surface)');
|
|
169
|
-
say('probe-end');
|