orizu 0.5.48 → 0.5.49
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/hosted-optimization.d.ts.map +1 -1
- package/dist/hosted-optimization.js +40 -21
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +83 -68
- package/dist/runner-folders.d.ts +61 -0
- package/dist/runner-folders.d.ts.map +1 -0
- package/dist/runner-folders.js +222 -0
- package/dist/runner-version-materialization.d.ts +1 -1
- package/dist/runner-version-materialization.d.ts.map +1 -1
- package/dist/runner-version-materialization.js +20 -38
- package/dist/skilled-proposer-launch.d.ts +11 -1
- package/dist/skilled-proposer-launch.d.ts.map +1 -1
- package/dist/skilled-proposer-launch.js +35 -4
- package/package.json +1 -1
- package/scripts/build-cli-dist.mjs +46 -8
- package/scripts/vendor-orizu-skill.mjs +52 -7
- package/src/hosted-optimization.ts +49 -31
- package/src/index.ts +79 -87
- package/src/runner-folders.ts +223 -0
- package/src/runner-version-materialization.ts +21 -34
- package/src/skilled-proposer-launch.ts +41 -3
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"hosted-optimization.d.ts","sourceRoot":"","sources":["../src/hosted-optimization.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"hosted-optimization.d.ts","sourceRoot":"","sources":["../src/hosted-optimization.ts"],"names":[],"mappings":"AAcA,UAAU,oBAAoB;IAAG,QAAQ,CAAC,EAAE,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,OAAO,CAAC,IAAI,CAAC,CAAA;CAAE;AAgZrF,wBAAsB,yBAAyB,CAAC,EAAE,EAAE,oBAAoB,GAAG,OAAO,CAAC,MAAM,CAAC,CAoHzF"}
|
|
@@ -3,11 +3,9 @@ import { existsSync, readFileSync, rmSync } from 'node:fs';
|
|
|
3
3
|
import { createServer } from 'node:http';
|
|
4
4
|
import { tmpdir } from 'node:os';
|
|
5
5
|
import { delimiter, join } from 'node:path';
|
|
6
|
-
import { runPipedChild } from './child-output-tee.js';
|
|
7
6
|
import { getGepaPythonPathEntries } from './gepa-python-paths.js';
|
|
8
7
|
import { deliveredProviderLookup, HOSTED_PROVIDER_FAILURE, INJECTED_ENV_VARS_ENV, PROVIDER_REGISTRY_EXTRA_ENV } from './hosted-environment.js';
|
|
9
8
|
import { credentialEnvFor, parseModelIdentity } from './provider-registry.js';
|
|
10
|
-
import { materializeRunnerVersion, cleanupMaterializedRunners } from './runner-version-materialization.js';
|
|
11
9
|
import { prepareSkilledProposerLaunch, spawnSkilledProposerChild } from './skilled-proposer-launch.js';
|
|
12
10
|
import { hostedProviderFromModel } from './hosted-provider-settings.js';
|
|
13
11
|
import { validNormalizedSkilledProposerConfig } from './skilled-proposer-wire.js';
|
|
@@ -331,22 +329,20 @@ async function startTokenBroker(boot, initial) {
|
|
|
331
329
|
}
|
|
332
330
|
return {
|
|
333
331
|
url: `http://127.0.0.1:${address.port}/${nonce}`,
|
|
334
|
-
|
|
332
|
+
// Also drop open keep-alive connections, so no refresh succeeds after stop.
|
|
333
|
+
stop: () => { server.close(); server.closeAllConnections(); },
|
|
335
334
|
token: () => activeToken,
|
|
336
335
|
secrets: () => [...tokens],
|
|
337
336
|
};
|
|
338
337
|
}
|
|
339
|
-
async function spawnOfficial(python, environment, ready, output) {
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
});
|
|
345
|
-
}
|
|
346
|
-
return runPipedChild(python, ['-m', 'orizu_gepa_connector'], environment, ready, {
|
|
338
|
+
async function spawnOfficial(python, environment, ready, output, stopOptions) {
|
|
339
|
+
// ORI-2429: both proposers start through the launcher (ORI-2402), so a stop
|
|
340
|
+
// signal reaches the optimizer, which then ends instead of outliving the
|
|
341
|
+
// CLI, and the CLI cleans up before it ends by that signal.
|
|
342
|
+
return spawnSkilledProposerChild(python, ['-m', 'orizu_gepa_connector'], 'official', environment, ready, {
|
|
347
343
|
stdout: chunk => output.stdout.append(chunk),
|
|
348
344
|
stderr: chunk => output.stderr.append(chunk),
|
|
349
|
-
});
|
|
345
|
+
}, stopOptions);
|
|
350
346
|
}
|
|
351
347
|
async function ensureFailedRun(boot, token, engineError, diagnostics) {
|
|
352
348
|
const headers = { authorization: `Bearer ${token}`, 'content-type': 'application/json' };
|
|
@@ -385,7 +381,13 @@ export async function hostedOptimizationCommand(io) {
|
|
|
385
381
|
}
|
|
386
382
|
delete process.env.ORIZU_BOOT_SECRET;
|
|
387
383
|
let reason = 'hosted_optimization_mint_refused';
|
|
388
|
-
|
|
384
|
+
// ORI-2429: one Effect scope owns the runner copies, the result file and the
|
|
385
|
+
// token broker, so a stop signal removes them too (see runner-folders.ts).
|
|
386
|
+
const { Effect, Scope } = await import('effect');
|
|
387
|
+
const { downloadedRunner, removeOnClose, RemovalLedger, stopClosesScope } = await import('./runner-folders.js');
|
|
388
|
+
const ledger = new RemovalLedger();
|
|
389
|
+
const { scope, close, stop, listen, removeHandlers } = stopClosesScope();
|
|
390
|
+
const inScope = (effect) => Effect.runPromise(effect.pipe(Effect.provideService(Scope.Scope, scope)));
|
|
389
391
|
let broker;
|
|
390
392
|
const resultFile = join(tmpdir(), `orizu-hosted-optimization-result-${boot.runId}-${randomUUID()}.json`);
|
|
391
393
|
const diagnosticSecrets = () => injectedSecrets(boot, broker);
|
|
@@ -393,6 +395,7 @@ export async function hostedOptimizationCommand(io) {
|
|
|
393
395
|
const bootOutput = new BoundedTextCapture(diagnosticSecrets);
|
|
394
396
|
let failureDiagnostics;
|
|
395
397
|
try {
|
|
398
|
+
listen();
|
|
396
399
|
const first = await mint(boot);
|
|
397
400
|
if (first.response.status === 422)
|
|
398
401
|
reason = 'hosted_optimization_spec_missing';
|
|
@@ -416,15 +419,23 @@ export async function hostedOptimizationCommand(io) {
|
|
|
416
419
|
process.env.ORIZU_BASE_URL = boot.baseUrl.origin;
|
|
417
420
|
process.env.ORIZU_TOKEN = minted.token;
|
|
418
421
|
reason = 'hosted_optimization_materialization_failed';
|
|
419
|
-
const candidate = await
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
cleanups.push(scorer.cleanup);
|
|
422
|
+
const candidate = await inScope(downloadedRunner(String(minted.jobSpec.runnerVersionId), ledger));
|
|
423
|
+
const scorer = await inScope(downloadedRunner(String(minted.jobSpec.scorerRunnerVersionId), ledger));
|
|
424
|
+
await inScope(removeOnClose(ledger, 'other', () => rmSync(resultFile, { force: true })));
|
|
423
425
|
broker = await startTokenBroker(boot, minted);
|
|
426
|
+
const startedBroker = broker;
|
|
427
|
+
await inScope(removeOnClose(ledger, 'other', () => startedBroker.stop()));
|
|
424
428
|
const environment = childEnvironment(boot, minted, candidate.runnerDir, scorer.runnerDir, broker.url, resultFile);
|
|
425
429
|
const launch = prepareSkilledProposerLaunch(process.env.PYTHON || 'python3', 'official', environment);
|
|
426
430
|
reason = 'hosted_optimization_process_failed';
|
|
427
|
-
|
|
431
|
+
stop.launcherOwnsStop();
|
|
432
|
+
let result;
|
|
433
|
+
try {
|
|
434
|
+
result = await spawnOfficial(launch.python, launch.environment, () => postBoot(boot, 'ready', { status: 'ready' }), processOutput, { beforeSignalExit: stop.beforeSignalExit });
|
|
435
|
+
}
|
|
436
|
+
finally {
|
|
437
|
+
stop.launcherDone();
|
|
438
|
+
}
|
|
428
439
|
if (result.error || result.status !== 0) {
|
|
429
440
|
const processFailure = result.error?.message || `optimizer exited with status ${result.status}`;
|
|
430
441
|
const secrets = injectedSecrets(boot, broker);
|
|
@@ -459,8 +470,16 @@ export async function hostedOptimizationCommand(io) {
|
|
|
459
470
|
return 1;
|
|
460
471
|
}
|
|
461
472
|
finally {
|
|
462
|
-
|
|
463
|
-
|
|
464
|
-
|
|
473
|
+
removeHandlers();
|
|
474
|
+
// Closing stops the broker, deletes the result file and removes the copies.
|
|
475
|
+
close();
|
|
476
|
+
// As before: a broker or result-file failure is thrown first, then runner removals.
|
|
477
|
+
const otherFailure = ledger.failures.find(failure => failure.group === 'other');
|
|
478
|
+
if (otherFailure)
|
|
479
|
+
throw otherFailure.error;
|
|
480
|
+
const runnerFailures = ledger.failures.filter(failure => failure.group === 'downloads').map(failure => failure.error);
|
|
481
|
+
if (runnerFailures.length > 0) {
|
|
482
|
+
throw new AggregateError(runnerFailures, 'Failed to clean materialized runner directories');
|
|
483
|
+
}
|
|
465
484
|
}
|
|
466
485
|
}
|
package/dist/index.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";
|
|
1
|
+
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";AAsLA,OAAO,EAAE,kBAAkB,EAAE,MAAM,6BAA6B,CAAA;AAEhE,eAAO,MAAM,wBAAwB,GAAI,iBAAiB,MAAM;;;EACJ,CAAA;AAE5D,OAAO,EAAE,iBAAiB,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAA;AAC5E,OAAO,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAA;AA+sCtD,wBAAgB,kBAAkB,CAAC,SAAS,EAAE,MAAM,GAAG;IAAE,OAAO,EAAE,MAAM,EAAE,CAAC;IAAC,kBAAkB,CAAC,EAAE,MAAM,EAAE,CAAA;CAAE,CA0B1G;AAED,eAAO,MAAM,iBAAiB,SAAU,CAAA;AACxC,eAAO,MAAM,uBAAuB,QAAkB,CAAA;AACtD,wBAAgB,mBAAmB,CAAC,SAAS,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,EAAE,iBAAiB,CAAC,EAAE,MAAM,GAAG,MAAM,CAAC,UAAU,CAexH;AAmMD,wBAAsB,IAAI,CAAC,OAAO,WAAwB,iBAiiBzD;AA8DD,OAAO,EAAE,kBAAkB,EAAE,cAAc,EAAE,MAAM,6BAA6B,CAAA;AAEhF,OAAO,EAAE,mBAAmB,EAAE,kBAAkB,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAA"}
|
package/dist/index.js
CHANGED
|
@@ -24,7 +24,6 @@ import { readMarkdownReportInput } from './markdown-report-input.js';
|
|
|
24
24
|
import { pushPromptDraft } from './prompt-draft-push.js';
|
|
25
25
|
import { promptPushErrorMessage, promptReportCommand } from './prompt-report-cli.js';
|
|
26
26
|
import { parseCommaSeparated, listTasks, createTask, updateTask, discardTask, publishTask, assignTask, taskStatus, setTaskReport, getTaskReport, updateTaskStatus, } from './task-commands.js';
|
|
27
|
-
import { cleanupAll, throwWithCleanup } from './cleanup.js';
|
|
28
27
|
import { extractFlagValue, verifyGepaRunnerDirsFromArgs, verifyRunnerDirRegistered, assertSnapshotManifestConfined, } from './runner-dir-verify.js';
|
|
29
28
|
import { getGepaPythonCommand } from './gepa-python-command.js';
|
|
30
29
|
import { resolveAuthTokenForBaseUrl } from './credentials.js';
|
|
@@ -941,82 +940,96 @@ const GEPA_RUNNER_MATERIALIZATION_PAIRS = [
|
|
|
941
940
|
{ versionFlag: '--runner-version-id', dirFlag: '--candidate-runner-dir' },
|
|
942
941
|
{ versionFlag: '--scorer-runner-version-id', dirFlag: '--scorer-runner-dir' },
|
|
943
942
|
];
|
|
944
|
-
async function materializeMissingGepaRunnerDirs(args) {
|
|
945
|
-
const rewritten = [...args];
|
|
946
|
-
const cleanups = [];
|
|
947
|
-
const cleanup = () => cleanupAll(cleanups, 'Failed to clean materialized runner directories');
|
|
948
|
-
try {
|
|
949
|
-
for (const pair of GEPA_RUNNER_MATERIALIZATION_PAIRS) {
|
|
950
|
-
const runnerVersionId = extractFlagValue(rewritten, pair.versionFlag);
|
|
951
|
-
const runnerDir = extractFlagValue(rewritten, pair.dirFlag);
|
|
952
|
-
if (!runnerVersionId || runnerDir)
|
|
953
|
-
continue;
|
|
954
|
-
const materialized = await materializeRunnerVersion(runnerVersionId);
|
|
955
|
-
cleanups.push(materialized.cleanup);
|
|
956
|
-
rewritten.push(pair.dirFlag, materialized.runnerDir);
|
|
957
|
-
}
|
|
958
|
-
}
|
|
959
|
-
catch (error) {
|
|
960
|
-
throwWithCleanup(error, cleanups, 'Runner materialization failed and cleanup also failed');
|
|
961
|
-
}
|
|
962
|
-
return { args: rewritten, cleanup };
|
|
963
|
-
}
|
|
964
943
|
async function runGepaOptimization() {
|
|
965
944
|
const project = getArg('--project') || await resolveProjectSlug(null);
|
|
966
945
|
const baseUrl = getBaseUrl();
|
|
967
946
|
const runGepa = getGepaPythonCommand(cliArgs.slice(2), process.env);
|
|
968
947
|
const python = runGepa.python;
|
|
969
|
-
|
|
970
|
-
//
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
//
|
|
975
|
-
|
|
976
|
-
const materialized = await materializeMissingGepaRunnerDirs(forwardedArgs);
|
|
977
|
-
let verified;
|
|
948
|
+
// ORI-2429: one Effect scope owns every runner copy this run writes to the
|
|
949
|
+
// temp folder; it removes them on success, failure and stop signals.
|
|
950
|
+
const { Effect } = await import('effect');
|
|
951
|
+
const { downloadedRunner, heldUntilClose, RemovalLedger, runStoppable } = await import('./runner-folders.js');
|
|
952
|
+
const ledger = new RemovalLedger();
|
|
953
|
+
// Which step failed decides the message when a removal also failed, as before.
|
|
954
|
+
let phase = 'materialize';
|
|
978
955
|
let selectedEngine = 'official';
|
|
956
|
+
let failure;
|
|
979
957
|
try {
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
|
|
983
|
-
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
958
|
+
await runStoppable(stop => Effect.gen(function* () {
|
|
959
|
+
// Download registered runner versions passed without a local folder; the
|
|
960
|
+
// verification below then checks those bytes like any local folder.
|
|
961
|
+
const materializedArgs = [...runGepa.args];
|
|
962
|
+
for (const pair of GEPA_RUNNER_MATERIALIZATION_PAIRS) {
|
|
963
|
+
const runnerVersionId = extractFlagValue(materializedArgs, pair.versionFlag);
|
|
964
|
+
const runnerDir = extractFlagValue(materializedArgs, pair.dirFlag);
|
|
965
|
+
if (!runnerVersionId || runnerDir)
|
|
966
|
+
continue;
|
|
967
|
+
const materialized = yield* downloadedRunner(runnerVersionId, ledger, false);
|
|
968
|
+
materializedArgs.push(pair.dirFlag, materialized.runnerDir);
|
|
969
|
+
}
|
|
970
|
+
phase = 'launch';
|
|
971
|
+
// ALI-1159 (ADR-007): GEPA executes ad-hoc local runner dirs while
|
|
972
|
+
// attributing records to registered runner version ids — verify the bytes
|
|
973
|
+
// ARE those versions before the optimizer spawns. Scans the forwarded argv
|
|
974
|
+
// itself (both argparse flag forms), fail-closed on half-populated pairs,
|
|
975
|
+
// and REWRITES the dir flags to verified snapshots (the exact hashed bytes)
|
|
976
|
+
// so a post-hash mutation of the original dirs cannot change what runs.
|
|
977
|
+
const verified = yield* heldUntilClose(ledger, 'snapshots', () => verifyGepaRunnerDirsFromArgs(materializedArgs));
|
|
978
|
+
let forwardedArgs = verified.args;
|
|
979
|
+
// Uniform resolution (ALI-1090): honors ORIZU_TOKEN / ORIZU_TOKEN_FILE before
|
|
980
|
+
// credentials.json so hosted sandboxes can run optimizations. Read at spawn
|
|
981
|
+
// time — AFTER the verification awaits above, so a hosted token-file
|
|
982
|
+
// rotation crossing the hash + lookup round-trips cannot hand the child a
|
|
983
|
+
// stale bearer.
|
|
984
|
+
const token = resolveAuthTokenForBaseUrl(baseUrl);
|
|
985
|
+
if (!forwardedArgs.includes('--project')) {
|
|
986
|
+
forwardedArgs = ['--project', project, ...forwardedArgs];
|
|
987
|
+
}
|
|
988
|
+
const dispatch = applyHostedInstructionSetProfileOverride(dispatchGepaEngine(forwardedArgs, project, {
|
|
989
|
+
...process.env,
|
|
990
|
+
ORIZU_API_URL: baseUrl,
|
|
991
|
+
ORIZU_TOKEN: token,
|
|
992
|
+
ORIZU_PROJECT: project,
|
|
993
|
+
ORIZU_VERIFIED_RUNNER_DIRS: JSON.stringify(verified.verifiedDirs),
|
|
994
|
+
PYTHONPATH: getGepaPythonPathEntries(process.env.PYTHONPATH).join(delimiter),
|
|
995
|
+
PYTHONUNBUFFERED: process.env.PYTHONUNBUFFERED || '1',
|
|
996
|
+
}));
|
|
997
|
+
const instructionSetProfileVersionId = yield* Effect.tryPromise({
|
|
998
|
+
try: () => resolveGepaSeedFromEnvironment(dispatch.environment, project, 'production'),
|
|
999
|
+
catch: error => error,
|
|
1000
|
+
});
|
|
1001
|
+
if (instructionSetProfileVersionId)
|
|
1002
|
+
dispatch.environment.ORIZU_INSTRUCTION_SET_PROFILE_VERSION_ID = instructionSetProfileVersionId;
|
|
1003
|
+
selectedEngine = dispatch.engine;
|
|
1004
|
+
const launch = prepareSkilledProposerLaunch(python, dispatch.engine, dispatch.environment);
|
|
1005
|
+
// While the optimizer runs, the launcher owns stop signals (ORI-2402) and
|
|
1006
|
+
// closes this scope just before the CLI ends by the signal. Taken in the
|
|
1007
|
+
// same step as the launch, so no stop can fall between the two.
|
|
1008
|
+
const result = yield* Effect.tryPromise({
|
|
1009
|
+
try: () => (stop.launcherOwnsStop(), spawnSkilledProposerChild(launch.python, ['-m', dispatch.module, ...dispatch.args], dispatch.engine, launch.environment, undefined, undefined, { beforeSignalExit: stop.beforeSignalExit })),
|
|
1010
|
+
catch: error => error,
|
|
1011
|
+
}).pipe(Effect.ensuring(Effect.sync(stop.launcherDone)));
|
|
1012
|
+
if (result.error) {
|
|
1013
|
+
return yield* Effect.fail(new Error(`${selectedEngine} GEPA engine failed: ${result.error.message}`, { cause: result.error }));
|
|
1014
|
+
}
|
|
1015
|
+
if (result.status !== 0) {
|
|
1016
|
+
return yield* Effect.fail(new Error(`${selectedEngine} GEPA engine failed with exit code ${result.status}`));
|
|
1017
|
+
}
|
|
999
1018
|
}));
|
|
1000
|
-
const instructionSetProfileVersionId = await resolveGepaSeedFromEnvironment(dispatch.environment, project, 'production');
|
|
1001
|
-
if (instructionSetProfileVersionId)
|
|
1002
|
-
dispatch.environment.ORIZU_INSTRUCTION_SET_PROFILE_VERSION_ID = instructionSetProfileVersionId;
|
|
1003
|
-
selectedEngine = dispatch.engine;
|
|
1004
|
-
const launch = prepareSkilledProposerLaunch(python, dispatch.engine, dispatch.environment);
|
|
1005
|
-
const result = await spawnSkilledProposerChild(launch.python, ['-m', dispatch.module, ...dispatch.args], dispatch.engine, launch.environment);
|
|
1006
|
-
if (result.error) {
|
|
1007
|
-
throw new Error(`${selectedEngine} GEPA engine failed: ${result.error.message}`, { cause: result.error });
|
|
1008
|
-
}
|
|
1009
|
-
if (result.status !== 0) {
|
|
1010
|
-
throw new Error(`${selectedEngine} GEPA engine failed with exit code ${result.status}`);
|
|
1011
|
-
}
|
|
1012
1019
|
}
|
|
1013
1020
|
catch (error) {
|
|
1014
|
-
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1021
|
+
failure = { error };
|
|
1022
|
+
}
|
|
1023
|
+
// The scope has closed here; a removal failure decides the message, as before.
|
|
1024
|
+
if (ledger.has()) {
|
|
1025
|
+
if (!failure)
|
|
1026
|
+
throw new AggregateError(ledger.errors(), 'Failed to clean GEPA runner directories');
|
|
1027
|
+
throw new AggregateError([failure.error, ...ledger.errors()], phase === 'materialize'
|
|
1028
|
+
? 'Runner materialization failed and cleanup also failed'
|
|
1029
|
+
: 'GEPA launch failed and runner cleanup also failed');
|
|
1030
|
+
}
|
|
1031
|
+
if (failure)
|
|
1032
|
+
throw failure.error;
|
|
1020
1033
|
if (hasJsonFlag()) {
|
|
1021
1034
|
printJson({ status: 'completed', exitCode: 0 });
|
|
1022
1035
|
}
|
|
@@ -1794,7 +1807,9 @@ function isCliEntrypoint() {
|
|
|
1794
1807
|
}
|
|
1795
1808
|
if (isCliEntrypoint()) {
|
|
1796
1809
|
const rawArgs = process.argv.slice(2);
|
|
1797
|
-
withPassiveUpdateNotice(rawArgs, getCliVersion, printError, args =>
|
|
1810
|
+
withPassiveUpdateNotice(rawArgs, getCliVersion, printError, args =>
|
|
1811
|
+
// ORI-2429: the process-boundary adapter stays a Promise (ADR-038 guide); the record write is Effect-owned and never replaces this error.
|
|
1812
|
+
main(args).catch(async (error) => {
|
|
1798
1813
|
const message = error instanceof Error ? error.message : 'Unknown error';
|
|
1799
1814
|
// The reporter email is unknown on the failure path, so the record masks every address.
|
|
1800
1815
|
await writeLastErrorRecord({
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { Effect, Scope } from 'effect';
|
|
2
|
+
export type RemovalGroup = 'downloads' | 'snapshots' | 'other';
|
|
3
|
+
/** Removal failures seen while a scope closes; Effect finalizers cannot fail. */
|
|
4
|
+
export declare class RemovalLedger {
|
|
5
|
+
readonly failures: {
|
|
6
|
+
group: RemovalGroup;
|
|
7
|
+
error: unknown;
|
|
8
|
+
}[];
|
|
9
|
+
has(group?: RemovalGroup): boolean;
|
|
10
|
+
errors(): unknown[];
|
|
11
|
+
}
|
|
12
|
+
/** Remove with `remove` when the surrounding scope closes, recording a failure. */
|
|
13
|
+
export declare function removeOnClose(ledger: RemovalLedger, group: RemovalGroup, remove: () => void): Effect.Effect<void, never, Scope.Scope>;
|
|
14
|
+
/**
|
|
15
|
+
* Download a registered runner version into a private temp folder that the
|
|
16
|
+
* caller's scope removes. A stop mid-download aborts the transfer and still
|
|
17
|
+
* removes the folder. Messages match the previous Promise implementation.
|
|
18
|
+
*/
|
|
19
|
+
export declare function downloadedRunner(runnerVersionId: string, ledger: RemovalLedger, verifyRegisteredDigest?: boolean): Effect.Effect<{
|
|
20
|
+
runnerDir: string;
|
|
21
|
+
remove: () => void;
|
|
22
|
+
}, unknown, Scope.Scope>;
|
|
23
|
+
/** A Promise-returning resource (acquired whole) whose cleanup the scope runs. */
|
|
24
|
+
export declare function heldUntilClose<A extends {
|
|
25
|
+
cleanup: () => void;
|
|
26
|
+
}>(ledger: RemovalLedger, group: RemovalGroup, acquire: () => Promise<A>): Effect.Effect<A, unknown, Scope.Scope>;
|
|
27
|
+
export interface StopControl {
|
|
28
|
+
/** Hand stop signals to the optimizer launcher; pass this as its beforeSignalExit. */
|
|
29
|
+
readonly beforeSignalExit: () => Promise<void> | void;
|
|
30
|
+
readonly launcherOwnsStop: () => void;
|
|
31
|
+
readonly launcherDone: () => void;
|
|
32
|
+
}
|
|
33
|
+
/**
|
|
34
|
+
* Run `program` in a scope that removes the runner copies however it ends.
|
|
35
|
+
* Before the optimizer starts, a stop signal interrupts the program (the scope
|
|
36
|
+
* closes as it unwinds) and the CLI then ends by that signal. While the
|
|
37
|
+
* launcher runs the optimizer, the launcher owns the signal and calls
|
|
38
|
+
* `beforeSignalExit`, which closes the scope synchronously: the removal then
|
|
39
|
+
* finishes even when the optimizer used up the whole stop cap.
|
|
40
|
+
*/
|
|
41
|
+
export declare function runStoppable<A>(program: (stop: StopControl) => Effect.Effect<A, unknown, Scope.Scope>): Promise<A>;
|
|
42
|
+
/**
|
|
43
|
+
* For a workflow that stays a Promise (hosted optimization): a scope that a
|
|
44
|
+
* stop signal closes before the CLI ends by that signal. Before the optimizer
|
|
45
|
+
* starts our handlers do it; while the launcher runs it, the launcher calls
|
|
46
|
+
* `stop.beforeSignalExit`. Closing aborts an in-flight download and removes
|
|
47
|
+
* every registered file, folder and server; the finalizers are synchronous.
|
|
48
|
+
*/
|
|
49
|
+
export declare function stopClosesScope(): {
|
|
50
|
+
scope: Scope.Closeable;
|
|
51
|
+
close: () => void;
|
|
52
|
+
stop: StopControl;
|
|
53
|
+
listen: () => void;
|
|
54
|
+
removeHandlers: () => void;
|
|
55
|
+
};
|
|
56
|
+
/** Open a scope for an adapter that must hand back a synchronous cleanup. */
|
|
57
|
+
export declare function openScope(): {
|
|
58
|
+
scope: Scope.Closeable;
|
|
59
|
+
close: () => void;
|
|
60
|
+
};
|
|
61
|
+
//# sourceMappingURL=runner-folders.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"runner-folders.d.ts","sourceRoot":"","sources":["../src/runner-folders.ts"],"names":[],"mappings":"AAIA,OAAO,EAAS,MAAM,EAAe,KAAK,EAAE,MAAM,QAAQ,CAAA;AAY1D,MAAM,MAAM,YAAY,GAAG,WAAW,GAAG,WAAW,GAAG,OAAO,CAAA;AAE9D,iFAAiF;AACjF,qBAAa,aAAa;IACxB,QAAQ,CAAC,QAAQ,EAAE;QAAE,KAAK,EAAE,YAAY,CAAC;QAAC,KAAK,EAAE,OAAO,CAAA;KAAE,EAAE,CAAK;IACjE,GAAG,CAAC,KAAK,CAAC,EAAE,YAAY,GAAG,OAAO;IAGlC,MAAM,IAAI,OAAO,EAAE;CAGpB;AAED,mFAAmF;AACnF,wBAAgB,aAAa,CAAC,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,MAAM,EAAE,MAAM,IAAI,2CAI3F;AAaD;;;;GAIG;AACH,wBAAgB,gBAAgB,CAAC,eAAe,EAAE,MAAM,EAAE,MAAM,EAAE,aAAa,EAAE,sBAAsB,UAAO;;;yBAgD7G;AAED,kFAAkF;AAClF,wBAAgB,cAAc,CAAC,CAAC,SAAS;IAAE,OAAO,EAAE,MAAM,IAAI,CAAA;CAAE,EAAE,MAAM,EAAE,aAAa,EAAE,KAAK,EAAE,YAAY,EAAE,OAAO,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,0CAStI;AAMD,MAAM,WAAW,WAAW;IAC1B,sFAAsF;IACtF,QAAQ,CAAC,gBAAgB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,GAAG,IAAI,CAAA;IACrD,QAAQ,CAAC,gBAAgB,EAAE,MAAM,IAAI,CAAA;IACrC,QAAQ,CAAC,YAAY,EAAE,MAAM,IAAI,CAAA;CAClC;AAED;;;;;;;GAOG;AACH,wBAAsB,YAAY,CAAC,CAAC,EAAE,OAAO,EAAE,CAAC,IAAI,EAAE,WAAW,KAAK,MAAM,CAAC,MAAM,CAAC,CAAC,EAAE,OAAO,EAAE,KAAK,CAAC,KAAK,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CA2CxH;AAED;;;;;;GAMG;AACH,wBAAgB,eAAe;;;;;;EAuB9B;AAED,6EAA6E;AAC7E,wBAAgB,SAAS;;;EAWxB"}
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { mkdtempSync, rmSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { tmpdir } from 'node:os';
|
|
4
|
+
import { join } from 'node:path';
|
|
5
|
+
import { Cause, Effect, Exit, Fiber, Scope } from 'effect';
|
|
6
|
+
import { ARTIFACT_MAX_BYTES } from './artifact-pull.js';
|
|
7
|
+
import { authedFetch } from './http.js';
|
|
8
|
+
import { unzipRunnerArchive } from './runner-version-materialization.js';
|
|
9
|
+
/** Removal failures seen while a scope closes; Effect finalizers cannot fail. */
|
|
10
|
+
export class RemovalLedger {
|
|
11
|
+
failures = [];
|
|
12
|
+
has(group) {
|
|
13
|
+
return this.failures.some(failure => !group || failure.group === group);
|
|
14
|
+
}
|
|
15
|
+
errors() {
|
|
16
|
+
return this.failures.map(failure => failure.error);
|
|
17
|
+
}
|
|
18
|
+
}
|
|
19
|
+
/** Remove with `remove` when the surrounding scope closes, recording a failure. */
|
|
20
|
+
export function removeOnClose(ledger, group, remove) {
|
|
21
|
+
return Effect.addFinalizer(() => Effect.sync(() => {
|
|
22
|
+
try {
|
|
23
|
+
remove();
|
|
24
|
+
}
|
|
25
|
+
catch (error) {
|
|
26
|
+
ledger.failures.push({ group, error });
|
|
27
|
+
}
|
|
28
|
+
}));
|
|
29
|
+
}
|
|
30
|
+
function request(path, signal) {
|
|
31
|
+
return Effect.tryPromise({
|
|
32
|
+
try: requestSignal => authedFetch(path, { signal: AbortSignal.any([requestSignal, signal]) }),
|
|
33
|
+
catch: error => error,
|
|
34
|
+
});
|
|
35
|
+
}
|
|
36
|
+
function text(response) {
|
|
37
|
+
return Effect.tryPromise({ try: () => response.text(), catch: error => error });
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* Download a registered runner version into a private temp folder that the
|
|
41
|
+
* caller's scope removes. A stop mid-download aborts the transfer and still
|
|
42
|
+
* removes the folder. Messages match the previous Promise implementation.
|
|
43
|
+
*/
|
|
44
|
+
export function downloadedRunner(runnerVersionId, ledger, verifyRegisteredDigest = true) {
|
|
45
|
+
return Effect.gen(function* () {
|
|
46
|
+
// Aborting on close also ends a body still streaming when a stop arrives.
|
|
47
|
+
const controller = yield* Effect.acquireRelease(Effect.sync(() => new AbortController()), controller => Effect.sync(() => controller.abort()));
|
|
48
|
+
let contentSha256;
|
|
49
|
+
if (verifyRegisteredDigest) {
|
|
50
|
+
const detailResponse = yield* request(`/api/cli/runner-versions/${encodeURIComponent(runnerVersionId)}`, controller.signal);
|
|
51
|
+
if (!detailResponse.ok)
|
|
52
|
+
return yield* Effect.fail(new Error(`Failed to resolve runner version: ${yield* text(detailResponse)}`));
|
|
53
|
+
const detail = (yield* Effect.tryPromise({ try: () => detailResponse.json(), catch: error => error }));
|
|
54
|
+
if (detail.runnerVersionId !== runnerVersionId || detail.status !== 'sealed'
|
|
55
|
+
|| typeof detail.contentSha256 !== 'string' || !/^[0-9a-f]{64}$/u.test(detail.contentSha256)) {
|
|
56
|
+
return yield* Effect.fail(new Error('Runner version detail is invalid'));
|
|
57
|
+
}
|
|
58
|
+
contentSha256 = detail.contentSha256;
|
|
59
|
+
}
|
|
60
|
+
const response = yield* request(`/api/cli/runner-versions/${encodeURIComponent(runnerVersionId)}/download`, controller.signal);
|
|
61
|
+
if (!response.ok)
|
|
62
|
+
return yield* Effect.fail(new Error(`Failed to download runner version: ${yield* text(response)}`));
|
|
63
|
+
// The previous implementation removed the folder as soon as this runner
|
|
64
|
+
// failed, naming the runner when that removal failed too. Keep both.
|
|
65
|
+
let removed = false;
|
|
66
|
+
let tempDir = '';
|
|
67
|
+
const remove = () => { if (!removed) {
|
|
68
|
+
removed = true;
|
|
69
|
+
rmSync(tempDir, { recursive: true, force: true });
|
|
70
|
+
} };
|
|
71
|
+
// Created and registered for removal in one uninterruptible step.
|
|
72
|
+
yield* Effect.acquireRelease(Effect.sync(() => { tempDir = mkdtempSync(join(tmpdir(), 'orizu-runner-version-')); }), () => Effect.sync(() => {
|
|
73
|
+
try {
|
|
74
|
+
remove();
|
|
75
|
+
}
|
|
76
|
+
catch (error) {
|
|
77
|
+
ledger.failures.push({ group: 'downloads', error });
|
|
78
|
+
}
|
|
79
|
+
}));
|
|
80
|
+
return yield* Effect.gen(function* () {
|
|
81
|
+
const zipBytes = new Uint8Array(yield* Effect.tryPromise({ try: () => response.arrayBuffer(), catch: error => error }));
|
|
82
|
+
if (zipBytes.byteLength > ARTIFACT_MAX_BYTES)
|
|
83
|
+
return yield* Effect.fail(new Error(`Runner artifact exceeds ${ARTIFACT_MAX_BYTES} bytes`));
|
|
84
|
+
if (contentSha256 && createHash('sha256').update(zipBytes).digest('hex') !== contentSha256) {
|
|
85
|
+
return yield* Effect.fail(new Error('Runner artifact digest mismatch'));
|
|
86
|
+
}
|
|
87
|
+
const zipPath = join(tempDir, 'runner.zip'), runnerDir = join(tempDir, 'runner');
|
|
88
|
+
yield* Effect.try({ try: () => { writeFileSync(zipPath, zipBytes); unzipRunnerArchive(zipPath, runnerDir); }, catch: error => error });
|
|
89
|
+
return { runnerDir, remove };
|
|
90
|
+
}).pipe(Effect.catch(error => Effect.suspend(() => {
|
|
91
|
+
try {
|
|
92
|
+
remove();
|
|
93
|
+
}
|
|
94
|
+
catch (cleanupError) {
|
|
95
|
+
return Effect.fail(new AggregateError([error, cleanupError], `Runner ${runnerVersionId} materialization failed and cleanup also failed`));
|
|
96
|
+
}
|
|
97
|
+
return Effect.fail(error);
|
|
98
|
+
})));
|
|
99
|
+
});
|
|
100
|
+
}
|
|
101
|
+
/** A Promise-returning resource (acquired whole) whose cleanup the scope runs. */
|
|
102
|
+
export function heldUntilClose(ledger, group, acquire) {
|
|
103
|
+
// Acquiring stays uninterruptible: a stop waits for it to finish, so a
|
|
104
|
+
// half-created snapshot is never orphaned; the scope then removes it.
|
|
105
|
+
return Effect.acquireRelease(Effect.tryPromise({ try: acquire, catch: error => error }), resource => Effect.sync(() => {
|
|
106
|
+
try {
|
|
107
|
+
resource.cleanup();
|
|
108
|
+
}
|
|
109
|
+
catch (error) {
|
|
110
|
+
ledger.failures.push({ group, error });
|
|
111
|
+
}
|
|
112
|
+
}));
|
|
113
|
+
}
|
|
114
|
+
const STOP_SIGNALS = ['SIGINT', 'SIGTERM', 'SIGHUP', 'SIGQUIT'];
|
|
115
|
+
// Matches the launcher's stop cap (ORI-2402): a stop never waits longer.
|
|
116
|
+
const STOP_CLEANUP_WAIT_MS = 15_000;
|
|
117
|
+
/**
|
|
118
|
+
* Run `program` in a scope that removes the runner copies however it ends.
|
|
119
|
+
* Before the optimizer starts, a stop signal interrupts the program (the scope
|
|
120
|
+
* closes as it unwinds) and the CLI then ends by that signal. While the
|
|
121
|
+
* launcher runs the optimizer, the launcher owns the signal and calls
|
|
122
|
+
* `beforeSignalExit`, which closes the scope synchronously: the removal then
|
|
123
|
+
* finishes even when the optimizer used up the whole stop cap.
|
|
124
|
+
*/
|
|
125
|
+
export async function runStoppable(program) {
|
|
126
|
+
const { scope, close } = openScope();
|
|
127
|
+
let launcherOwned = false;
|
|
128
|
+
let stopping = false;
|
|
129
|
+
// eslint-disable-next-line prefer-const -- assigned after the handlers that read it
|
|
130
|
+
let fiber;
|
|
131
|
+
const interrupt = () => Effect.runPromise(Fiber.interrupt(fiber));
|
|
132
|
+
const removeHandlers = () => { for (const entry of handlers)
|
|
133
|
+
process.off(entry.signal, entry.handle); };
|
|
134
|
+
const handlers = STOP_SIGNALS.map(signal => ({
|
|
135
|
+
signal,
|
|
136
|
+
handle: () => {
|
|
137
|
+
if (launcherOwned || stopping)
|
|
138
|
+
return;
|
|
139
|
+
stopping = true;
|
|
140
|
+
let timer;
|
|
141
|
+
const expired = new Promise(resolve => { timer = setTimeout(resolve, STOP_CLEANUP_WAIT_MS); });
|
|
142
|
+
void Promise.race([interrupt(), expired]).finally(() => {
|
|
143
|
+
clearTimeout(timer);
|
|
144
|
+
removeHandlers();
|
|
145
|
+
process.kill(process.pid, signal);
|
|
146
|
+
});
|
|
147
|
+
},
|
|
148
|
+
}));
|
|
149
|
+
const control = {
|
|
150
|
+
beforeSignalExit: () => {
|
|
151
|
+
stopping = true;
|
|
152
|
+
// The launcher re-sends the signal next; none of ours may catch it.
|
|
153
|
+
removeHandlers();
|
|
154
|
+
close();
|
|
155
|
+
},
|
|
156
|
+
launcherOwnsStop: () => { launcherOwned = true; },
|
|
157
|
+
launcherDone: () => { launcherOwned = false; },
|
|
158
|
+
};
|
|
159
|
+
for (const entry of handlers)
|
|
160
|
+
process.on(entry.signal, entry.handle);
|
|
161
|
+
fiber = Effect.runFork(Scope.provide(scope)(program(control)).pipe(Effect.ensuring(Effect.sync(close))));
|
|
162
|
+
try {
|
|
163
|
+
const exit = await Effect.runPromise(Effect.exit(Fiber.join(fiber)));
|
|
164
|
+
// A stop ends the process by its signal; nothing more to report here.
|
|
165
|
+
if (stopping)
|
|
166
|
+
return await new Promise(() => { });
|
|
167
|
+
if (Exit.isSuccess(exit))
|
|
168
|
+
return exit.value;
|
|
169
|
+
throw Cause.squash(exit.cause);
|
|
170
|
+
}
|
|
171
|
+
finally {
|
|
172
|
+
removeHandlers();
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
/**
|
|
176
|
+
* For a workflow that stays a Promise (hosted optimization): a scope that a
|
|
177
|
+
* stop signal closes before the CLI ends by that signal. Before the optimizer
|
|
178
|
+
* starts our handlers do it; while the launcher runs it, the launcher calls
|
|
179
|
+
* `stop.beforeSignalExit`. Closing aborts an in-flight download and removes
|
|
180
|
+
* every registered file, folder and server; the finalizers are synchronous.
|
|
181
|
+
*/
|
|
182
|
+
export function stopClosesScope() {
|
|
183
|
+
const { scope, close } = openScope();
|
|
184
|
+
let launcherOwned = false;
|
|
185
|
+
let stopping = false;
|
|
186
|
+
const removeHandlers = () => { for (const entry of handlers)
|
|
187
|
+
process.off(entry.signal, entry.handle); };
|
|
188
|
+
const handlers = STOP_SIGNALS.map(signal => ({
|
|
189
|
+
signal,
|
|
190
|
+
handle: () => {
|
|
191
|
+
if (launcherOwned || stopping)
|
|
192
|
+
return;
|
|
193
|
+
stopping = true;
|
|
194
|
+
close();
|
|
195
|
+
removeHandlers();
|
|
196
|
+
process.kill(process.pid, signal);
|
|
197
|
+
},
|
|
198
|
+
}));
|
|
199
|
+
// Installed by the caller inside its try, so a setup throw cannot leave them behind.
|
|
200
|
+
const listen = () => { for (const entry of handlers)
|
|
201
|
+
process.on(entry.signal, entry.handle); };
|
|
202
|
+
const stop = {
|
|
203
|
+
beforeSignalExit: () => { stopping = true; removeHandlers(); close(); },
|
|
204
|
+
launcherOwnsStop: () => { launcherOwned = true; },
|
|
205
|
+
launcherDone: () => { launcherOwned = false; },
|
|
206
|
+
};
|
|
207
|
+
return { scope, close, stop, listen, removeHandlers };
|
|
208
|
+
}
|
|
209
|
+
/** Open a scope for an adapter that must hand back a synchronous cleanup. */
|
|
210
|
+
export function openScope() {
|
|
211
|
+
const scope = Scope.makeUnsafe();
|
|
212
|
+
let closed = false;
|
|
213
|
+
return {
|
|
214
|
+
scope,
|
|
215
|
+
close: () => {
|
|
216
|
+
if (closed)
|
|
217
|
+
return;
|
|
218
|
+
closed = true;
|
|
219
|
+
Effect.runSync(Scope.close(scope, Exit.void));
|
|
220
|
+
},
|
|
221
|
+
};
|
|
222
|
+
}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
|
+
export declare function unzipRunnerArchive(zipPath: string, runnerDir: string): void;
|
|
1
2
|
export declare function materializeRunnerVersion(runnerVersionId: string, verifyRegisteredDigest?: boolean): Promise<{
|
|
2
3
|
runnerDir: string;
|
|
3
4
|
cleanup: () => void;
|
|
4
5
|
}>;
|
|
5
|
-
export declare function cleanupMaterializedRunners(cleanups: Array<() => void>): void;
|
|
6
6
|
//# sourceMappingURL=runner-version-materialization.d.ts.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"runner-version-materialization.d.ts","sourceRoot":"","sources":["../src/runner-version-materialization.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"runner-version-materialization.d.ts","sourceRoot":"","sources":["../src/runner-version-materialization.ts"],"names":[],"mappings":"AAEA,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,IAAI,CAa3E;AACD,wBAAsB,wBAAwB,CAAC,eAAe,EAAE,MAAM,EAAE,sBAAsB,UAAO,GAAG,OAAO,CAAC;IAC9G,SAAS,EAAE,MAAM,CAAC;IAAC,OAAO,EAAE,MAAM,IAAI,CAAA;CACvC,CAAC,CAwBD"}
|