@aiwg/cli 2026.10.5 → 2026.10.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/agentic/code/addons/composition-engine/lib/runtime.mjs +1401 -0
- package/agentic/code/addons/composition-engine/lib/validator.mjs +513 -0
- package/agentic/code/addons/composition-engine/schemas/flow-graph.schema.json +372 -0
- package/agentic/code/providers/model-capabilities.v1.json +74 -50
- package/bin/aiwg.mjs +7 -3
- package/dist/src/a2a/agent-card.js +11 -0
- package/dist/src/api/index.d.ts +1 -0
- package/dist/src/api/index.js +1 -0
- package/dist/src/artifacts/browser-export.js +1 -0
- package/dist/src/artifacts/citation-parser.js +8 -1
- package/dist/src/artifacts/cli.js +57 -6
- package/dist/src/artifacts/fortemi-core-query-adapter.js +2 -0
- package/dist/src/artifacts/fortemi-core-sync.js +26 -8
- package/dist/src/artifacts/fortemi-shard-contract/2.2.0/contract.json +103 -0
- package/dist/src/artifacts/fortemi-shard-contract/2.2.0/full-v1-embedding-config.schema.json +212 -0
- package/dist/src/artifacts/fortemi-shard-contract/2.2.0/full-v1-embedding-set.schema.json +240 -0
- package/dist/src/artifacts/fortemi-shard-contract/2.2.0/full-v1-manifest.schema.json +451 -0
- package/dist/src/artifacts/fortemi-shard-export.js +284 -0
- package/dist/src/artifacts/graph-query.js +18 -8
- package/dist/src/artifacts/index-adapters.js +339 -0
- package/dist/src/artifacts/index-builder.js +143 -14
- package/dist/src/artifacts/index-files.js +15 -4
- package/dist/src/artifacts/index-preparation.js +2 -0
- package/dist/src/artifacts/index-reader.js +44 -0
- package/dist/src/artifacts/index-refresh.js +114 -55
- package/dist/src/artifacts/index-status.js +20 -5
- package/dist/src/artifacts/memory-maintenance.js +503 -0
- package/dist/src/artifacts/prebuilt-relations.js +3 -1
- package/dist/src/artifacts/query-engine.js +263 -108
- package/dist/src/artifacts/relations-builder.js +2 -2
- package/dist/src/artifacts/source-graph.js +2 -2
- package/dist/src/artifacts/types.js +9 -1
- package/dist/src/auth/client.js +5 -1
- package/dist/src/auth/credential-store.js +5 -1
- package/dist/src/auth/sensitive-input.js +151 -0
- package/dist/src/channel/manager.mjs +3 -0
- package/dist/src/cli/handlers/agentcard.js +47 -25
- package/dist/src/cli/handlers/auth.js +22 -2
- package/dist/src/cli/handlers/decision.js +22 -3
- package/dist/src/cli/handlers/effect.js +4 -84
- package/dist/src/cli/handlers/guided-setup.js +470 -0
- package/dist/src/cli/handlers/help.js +2 -2
- package/dist/src/cli/handlers/housekeeper.js +1110 -0
- package/dist/src/cli/handlers/index.js +4 -2
- package/dist/src/cli/handlers/init.js +41 -6
- package/dist/src/cli/handlers/install.js +6 -0
- package/dist/src/cli/handlers/local-executor.js +5 -3
- package/dist/src/cli/handlers/marketplace.js +15 -4
- package/dist/src/cli/handlers/mc.js +509 -53
- package/dist/src/cli/handlers/models.js +32 -2
- package/dist/src/cli/handlers/ralph-launcher.js +76 -4
- package/dist/src/cli/handlers/rules.js +1 -1
- package/dist/src/cli/handlers/script-runner.js +73 -2
- package/dist/src/cli/handlers/serve.js +5 -3
- package/dist/src/cli/handlers/sessions.js +1 -0
- package/dist/src/cli/handlers/setup.js +432 -1
- package/dist/src/cli/handlers/skill-lint.js +1 -1
- package/dist/src/cli/handlers/subcommands.js +118 -8
- package/dist/src/cli/handlers/use.js +335 -107
- package/dist/src/cli/handlers/workspace.js +3 -1
- package/dist/src/cli/prompt-utils.js +81 -0
- package/dist/src/cli/router.js +16 -1
- package/dist/src/config/aiwg-config.js +42 -0
- package/dist/src/config/cli.js +7 -3
- package/dist/src/config/decision-config.js +14 -1
- package/dist/src/config/gitignore.js +4 -0
- package/dist/src/config/native-sandbox-warnings.js +25 -0
- package/dist/src/config/workspace.js +2 -1
- package/dist/src/contributors/validation.js +6 -5
- package/dist/src/decision/adapters/jev.js +20 -31
- package/dist/src/decision/admission.js +7 -3
- package/dist/src/decision/ask.js +6 -1
- package/dist/src/decision/benchmark.js +482 -0
- package/dist/src/decision/ledger.js +22 -3
- package/dist/src/decision/pricing.js +147 -0
- package/dist/src/decision/qualification/gate-evidence.js +23 -18
- package/dist/src/decision/qualification/load-staged.js +576 -0
- package/dist/src/decision/qualification/load.js +109 -33
- package/dist/src/decision/qualification/runner.js +12 -5
- package/dist/src/decision/review/service.js +6 -4
- package/dist/src/decision/review/validate.js +23 -2
- package/dist/src/decision/savings.js +8 -3
- package/dist/src/discovery/index.js +492 -0
- package/dist/src/discovery/interview.js +238 -0
- package/dist/src/discovery/plan-apply.js +567 -0
- package/dist/src/discovery/plan-store.js +171 -0
- package/dist/src/discovery/plan-types.js +2 -0
- package/dist/src/discovery/plan-validation.js +139 -0
- package/dist/src/discovery/types.js +2 -0
- package/dist/src/effects/host.js +91 -0
- package/dist/src/extensions/commands/definitions.js +49 -9
- package/dist/src/extensions/manifest.js +71 -44
- package/dist/src/extensions/project-local-discovery.js +1 -1
- package/dist/src/extensions/project-local-gitignore.js +2 -0
- package/dist/src/extensions/project-quickref.js +18 -17
- package/dist/src/extensions/validation.js +32 -29
- package/dist/src/housekeeper/daemon-config.js +148 -0
- package/dist/src/housekeeper/ipc.js +84 -0
- package/dist/src/housekeeper/lifecycle.js +148 -0
- package/dist/src/housekeeper/platform.js +35 -0
- package/dist/src/housekeeper/status.js +70 -0
- package/dist/src/housekeeper/units.js +114 -0
- package/dist/src/issues/cli.js +16 -5
- package/dist/src/issues/live.js +44 -12
- package/dist/src/jobs/flow.js +22 -21
- package/dist/src/lint/rule-activation.js +20 -5
- package/dist/src/lint/runner.js +1 -1
- package/dist/src/marketplace/exchange.js +18 -6
- package/dist/src/marketplace/provenance.js +3 -2
- package/dist/src/mcp/adapters/codex-runtime.js +46 -76
- package/dist/src/mcp/cli.mjs +22 -96
- package/dist/src/mcp/config-file.mjs +6 -2
- package/dist/src/mcp/config-layers.mjs +23 -7
- package/dist/src/mcp/grok-build-config.mjs +2 -1
- package/dist/src/mcp/muse-native-config.mjs +2 -1
- package/dist/src/mcp/profiles.js +18 -31
- package/dist/src/mcp/profiles.mjs +18 -32
- package/dist/src/mcp/registry.js +7 -27
- package/dist/src/mcp/registry.mjs +7 -28
- package/dist/src/mcp/server.mjs +34 -1
- package/dist/src/mcp/tools/agentic-sandbox.mjs +8 -7
- package/dist/src/mcp/tools/command-run.mjs +3 -2
- package/dist/src/mcp/tools/decision.mjs +49 -48
- package/dist/src/mcp/tools/discovery.mjs +46 -21
- package/dist/src/mcp/tools/orchestration.mjs +22 -22
- package/dist/src/mcp/tools/subsystems.mjs +9 -8
- package/dist/src/mcp/tools/worker-invoke.mjs +30 -6
- package/dist/src/mcp/tools/writer-profiles.mjs +1 -1
- package/dist/src/memory/audit.js +719 -0
- package/dist/src/memory/canonical-context.js +392 -13
- package/dist/src/memory/canonical-maintenance.js +95 -0
- package/dist/src/memory/cli-result.js +22 -0
- package/dist/src/memory/cli.js +176 -0
- package/dist/src/memory/context-pack.js +57 -6
- package/dist/src/memory/cost.js +512 -0
- package/dist/src/memory/firewall-classification.d.mts +7 -0
- package/dist/src/memory/firewall-classification.mjs +32 -0
- package/dist/src/memory/index.js +8 -0
- package/dist/src/memory/line-maintenance.js +68 -0
- package/dist/src/memory/maintenance-cli.js +286 -0
- package/dist/src/memory/maintenance-contract.js +136 -0
- package/dist/src/memory/maintenance-index.js +18 -0
- package/dist/src/memory/maintenance-owner.js +2 -0
- package/dist/src/memory/maintenance-record.js +138 -0
- package/dist/src/memory/maintenance.js +254 -0
- package/dist/src/memory/project-registry.js +2 -1
- package/dist/src/memory/provider-memory-adapter.js +154 -0
- package/dist/src/memory/recommend-cli.js +83 -0
- package/dist/src/memory/recommend.js +269 -0
- package/dist/src/memory/scheduled.js +81 -0
- package/dist/src/memory/taxonomy.js +44 -0
- package/dist/src/mission-protocol/codecs.js +19 -9
- package/dist/src/mission-protocol/worker-status.js +91 -1
- package/dist/src/models/config-loader.js +11 -1
- package/dist/src/models/model-capabilities.v1.json +74 -50
- package/dist/src/models/provider-policy.js +125 -62
- package/dist/src/models/resolver.js +45 -8
- package/dist/src/models/worker-installation.d.mts +8 -1
- package/dist/src/models/worker-installation.mjs +231 -38
- package/dist/src/models/wrapper-route.js +1 -1
- package/dist/src/output-modes/registry.js +8 -7
- package/dist/src/packages/adapters/git.js +99 -23
- package/dist/src/packages/adapters/gitea.js +8 -27
- package/dist/src/packages/adapters/github.js +7 -25
- package/dist/src/packages/registry.js +4 -4
- package/dist/src/plugin/metadata-validator.js +1 -1
- package/dist/src/process/cli-command.d.mts +5 -0
- package/dist/src/process/cli-command.mjs +42 -0
- package/dist/src/providers/deploy-version.d.mts +1 -0
- package/dist/src/providers/deploy-version.mjs +17 -0
- package/dist/src/providers/provider-definitions.js +19 -16
- package/dist/src/providers/skill-ownership.d.mts +11 -2
- package/dist/src/providers/skill-ownership.mjs +35 -17
- package/dist/src/rules/activation.d.mts +11 -2
- package/dist/src/rules/activation.mjs +82 -6
- package/dist/src/rules/inventory.js +1 -1
- package/dist/src/rules/rule-frontmatter.schema.json +2 -0
- package/dist/src/schema/zod-validation.d.mts +6 -0
- package/dist/src/schema/zod-validation.mjs +50 -0
- package/dist/src/serve/executor-registry.js +297 -20
- package/dist/src/serve/executor-token.d.mts +1 -0
- package/dist/src/serve/executor-token.mjs +8 -0
- package/dist/src/serve/shared-host-scheduler.js +1102 -31
- package/dist/src/serve/worker-invocation.js +124 -15
- package/dist/src/sessions/adapters/claude.js +19 -18
- package/dist/src/sessions/adapters/codex.js +10 -9
- package/dist/src/sessions/adapters/copilot.js +14 -13
- package/dist/src/sessions/adapters/cursor.js +19 -18
- package/dist/src/sessions/adapters/deepseek-harness.js +9 -8
- package/dist/src/sessions/adapters/factory.js +11 -10
- package/dist/src/sessions/adapters/generic.js +32 -23
- package/dist/src/sessions/adapters/grokbot.js +35 -35
- package/dist/src/sessions/adapters/hermes.js +20 -19
- package/dist/src/sessions/adapters/muse.js +50 -49
- package/dist/src/sessions/adapters/omp.js +5 -4
- package/dist/src/sessions/adapters/openclaw.js +30 -29
- package/dist/src/sessions/adapters/opencode.js +26 -26
- package/dist/src/sessions/adapters/openhuman.js +24 -23
- package/dist/src/sessions/adapters/pi.js +5 -4
- package/dist/src/sessions/adapters/windsurf.js +8 -8
- package/dist/src/sessions/analytics.js +10 -9
- package/dist/src/sessions/candidates.js +7 -6
- package/dist/src/sessions/contracts.js +63 -53
- package/dist/src/sessions/output-registration.js +11 -10
- package/dist/src/sessions/workspace-discovery.js +3 -4
- package/dist/src/smiths/context-pipeline/aiwg-md.js +18 -1
- package/dist/src/smiths/context-pipeline/claude-hook.js +46 -10
- package/dist/src/smiths/context-pipeline/context-file-safety.js +330 -9
- package/dist/src/smiths/context-pipeline/finalization.js +3 -2
- package/dist/src/smiths/context-pipeline/generator.js +7 -19
- package/dist/src/smiths/context-pipeline/legacy-inject.js +3 -10
- package/dist/src/smiths/context-pipeline/managed-hook.js +5 -5
- package/dist/src/smiths/context-pipeline/parallelism-section.js +72 -5
- package/dist/src/smiths/context-pipeline/workspace-context.js +110 -22
- package/dist/src/tracker/project-inputs.js +2 -0
- package/dist/src/update/checker.d.mts +4 -0
- package/dist/src/update/checker.mjs +14 -0
- package/dist/src/update/notifier.mjs +2 -4
- package/dist/src/writing/exemplar-selection.js +6 -6
- package/dist/src/writing/voice-evaluation.js +24 -23
- package/dist/src/writing/writer-migration.js +14 -13
- package/dist/src/writing/writer-profile.js +19 -18
- package/dist/src/writing/writing-brief.js +18 -17
- package/dist/src/writing/writing-receipt.js +47 -46
- package/package.json +5 -2
- package/schemas/decision/DecisionReview.schema.json +78 -1
- package/schemas/mission-v1.schema.json +3 -0
- package/tools/agents/deploy-agents.mjs +67 -39
- package/tools/agents/providers/antigravity.mjs +6 -1
- package/tools/agents/providers/base.mjs +24 -14
- package/tools/agents/providers/claude.mjs +6 -3
- package/tools/agents/providers/codex.mjs +54 -57
- package/tools/agents/providers/context-write.mjs +28 -0
- package/tools/agents/providers/copilot.mjs +13 -4
- package/tools/agents/providers/cursor.mjs +1 -1
- package/tools/agents/providers/deepseek-harness.mjs +1 -1
- package/tools/agents/providers/factory.mjs +7 -1
- package/tools/agents/providers/grok-build.mjs +18 -2
- package/tools/agents/providers/grokbot.mjs +1 -1
- package/tools/agents/providers/hermes.mjs +12 -15
- package/tools/agents/providers/muse-hooks.mjs +21 -14
- package/tools/agents/providers/muse.mjs +7 -2
- package/tools/agents/providers/omp.mjs +31 -5
- package/tools/agents/providers/opencode.mjs +14 -5
- package/tools/agents/providers/openhuman.mjs +1 -1
- package/tools/agents/providers/pi.mjs +1 -1
- package/tools/agents/providers/provider-prepare-coordinator.mjs +29 -9
- package/tools/agents/providers/rule-delivery.mjs +289 -80
- package/tools/agents/providers/warp.mjs +4 -1
- package/tools/agents/providers/windsurf.mjs +13 -3
- package/tools/cli/workspace-status.mjs +5 -0
- package/tools/commands/deploy-prompts-codex.mjs +11 -8
- package/tools/ralph-external/lib/codex-adapter.mjs +6 -2
- package/tools/ralph-external/lib/consecutive-failure-circuit-breaker.mjs +26 -0
- package/tools/ralph-external/lib/launcher-recovery-evidence.mjs +7 -1
- package/tools/ralph-external/lib/muse-adapter.mjs +12 -2
- package/tools/ralph-external/lib/recovery-probe.mjs +14 -4
- package/tools/ralph-external/lib/worker-health.mjs +1 -1
- package/tools/ralph-external/lib/worker-isolation.mjs +65 -9
- package/tools/ralph-external/process-monitor.mjs +49 -8
- package/tools/ralph-external/session-launcher.mjs +405 -35
- package/tools/skills/deploy-skills-codex.mjs +11 -7
- package/vscode-extension/schemas/aiwg.config.v1.json +11 -1
|
@@ -8,9 +8,115 @@
|
|
|
8
8
|
*
|
|
9
9
|
* @implements #1566
|
|
10
10
|
*/
|
|
11
|
-
import { closeSync, existsSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync, } from 'node:fs';
|
|
12
|
-
import { dirname } from 'node:path';
|
|
13
|
-
import {
|
|
11
|
+
import { closeSync, constants, fstatSync, fsyncSync, lstatSync, existsSync, mkdirSync, openSync, readFileSync, realpathSync, renameSync, unlinkSync, writeFileSync, } from 'node:fs';
|
|
12
|
+
import { dirname, isAbsolute, join, relative, resolve } from 'node:path';
|
|
13
|
+
import { hostname } from 'node:os';
|
|
14
|
+
import { createHash } from 'node:crypto';
|
|
15
|
+
import { advanceProgressWatermark, assertMissionWorkerHandoff, assertValidWorkerPayload, decideLeaseMapping, projectHealthTrace } from '../mission-protocol/codecs.js';
|
|
16
|
+
import { isCanonicalWorkerModelKey } from '../models/provider-policy.js';
|
|
17
|
+
/** Translate existing owner ceilings without creating another budget contract. */
|
|
18
|
+
export function managedAllowanceFromOwner(runId, taskId, owner) {
|
|
19
|
+
const budgets = owner.kind === 'mission' ? owner.budgets : owner.kind === 'ralph' ? {
|
|
20
|
+
maxAttempts: owner.maxIterations, maxTokens: owner.maxTotalTokens, maxCostUsd: owner.maxTotalCost,
|
|
21
|
+
timeoutSeconds: owner.maxWallClockMinutes === undefined ? undefined : owner.maxWallClockMinutes * 60,
|
|
22
|
+
} : {
|
|
23
|
+
maxAttempts: owner.ceilings.activations, maxTokens: owner.ceilings.tokens, maxCostUsd: owner.ceilings.costUsd,
|
|
24
|
+
timeoutSeconds: owner.ceilings.timeMs === undefined ? undefined : owner.ceilings.timeMs / 1000,
|
|
25
|
+
};
|
|
26
|
+
return owner.kind === 'ralph'
|
|
27
|
+
? { runId, taskId, taskBudgets: budgets, attempts: owner.maxIterations ?? 5 }
|
|
28
|
+
: { runId, taskId, runBudgets: budgets };
|
|
29
|
+
}
|
|
30
|
+
/** Canonical project identity and the existing MC authority, including symlink aliases. */
|
|
31
|
+
export function projectAdmissionAuthority(projectRoot) {
|
|
32
|
+
const root = realpathSync(projectRoot);
|
|
33
|
+
return {
|
|
34
|
+
projectId: createHash('sha256').update(root).digest('hex'),
|
|
35
|
+
path: join(root, '.aiwg/ralph-external/mc/admission.json'),
|
|
36
|
+
};
|
|
37
|
+
}
|
|
38
|
+
export function managedDimensionKeys(projectId, worker) {
|
|
39
|
+
if (!/^[a-f0-9]{64}$/.test(projectId))
|
|
40
|
+
throw new Error('invalid canonical project identity');
|
|
41
|
+
for (const id of [worker.group, worker.member, worker.cli]) {
|
|
42
|
+
if (id !== undefined && !/^[a-z][a-z0-9-]{0,62}$/.test(id))
|
|
43
|
+
throw new Error('invalid canonical worker identity');
|
|
44
|
+
}
|
|
45
|
+
if (worker.member !== undefined && worker.group === undefined)
|
|
46
|
+
throw new Error('member requires a group');
|
|
47
|
+
if (worker.model !== undefined && !isCanonicalWorkerModelKey(worker.model))
|
|
48
|
+
throw new Error('invalid canonical model key');
|
|
49
|
+
if (worker.quotaScope !== undefined && !worker.quotaScope.trim())
|
|
50
|
+
throw new Error('quotaScope must be nonempty');
|
|
51
|
+
const project = `project/${projectId}`;
|
|
52
|
+
return [project,
|
|
53
|
+
...(worker.group ? [`${project}/group/${worker.group}`] : []),
|
|
54
|
+
...(worker.member ? [`${project}/group/${worker.group}/member/${worker.member}`] : []),
|
|
55
|
+
...(worker.cli ? [`${project}/installation/${worker.cli}`] : []),
|
|
56
|
+
...(worker.model ? [`model/${worker.model}`] : []),
|
|
57
|
+
...(worker.quotaScope ? [`quota/${encodeURIComponent(worker.quotaScope)}`] : []),
|
|
58
|
+
];
|
|
59
|
+
}
|
|
60
|
+
/** Build the project policy for the existing config/activation owner. */
|
|
61
|
+
export function managedProjectAdmissionPolicy(projectRoot, parallelism, policyDigest, maxConcurrent) {
|
|
62
|
+
const { projectId } = projectAdmissionAuthority(projectRoot);
|
|
63
|
+
const admission = parallelism.admission;
|
|
64
|
+
if ((admission?.authorityRef ?? 'project') !== 'project')
|
|
65
|
+
throw new Error('only project admission authority is supported by the project policy builder');
|
|
66
|
+
if (parallelism.budgets?.enforcement === 'hard')
|
|
67
|
+
throw new Error('OG_BUDGET_UNMEASURABLE: managed admission does not enforce hard spend budgets');
|
|
68
|
+
const dimensions = {};
|
|
69
|
+
const add = (worker, cap) => {
|
|
70
|
+
const key = managedDimensionKeys(projectId, worker).at(-1);
|
|
71
|
+
dimensions[key] = Math.min(dimensions[key] ?? cap, cap);
|
|
72
|
+
};
|
|
73
|
+
for (const [group, definition] of Object.entries(parallelism.groups ?? {})) {
|
|
74
|
+
if (definition.maxParallel !== undefined)
|
|
75
|
+
add({ kind: 'native', group }, definition.maxParallel);
|
|
76
|
+
}
|
|
77
|
+
for (const [group, cap] of Object.entries(admission?.groupCaps ?? {}))
|
|
78
|
+
add({ kind: 'native', group }, cap);
|
|
79
|
+
for (const [key, cap] of Object.entries(admission?.memberCaps ?? {})) {
|
|
80
|
+
const [group, member, extra] = key.split('/');
|
|
81
|
+
if (!group || !member || extra !== undefined)
|
|
82
|
+
throw new Error('invalid canonical group/member key');
|
|
83
|
+
add({ kind: 'native', group, member }, cap);
|
|
84
|
+
}
|
|
85
|
+
for (const [cli, cap] of Object.entries(admission?.cliCaps ?? {}))
|
|
86
|
+
add({ kind: 'native', cli }, cap);
|
|
87
|
+
for (const [model, cap] of Object.entries(admission?.modelCaps ?? {}))
|
|
88
|
+
add({ kind: 'native', model }, cap);
|
|
89
|
+
for (const [quotaScope, cap] of Object.entries(admission?.quotaCaps ?? {}))
|
|
90
|
+
add({ kind: 'native', quotaScope }, cap);
|
|
91
|
+
return {
|
|
92
|
+
maxConcurrent, leaseTtlMs: (admission?.leaseSeconds ?? 60) * 1000,
|
|
93
|
+
agingIntervalMs: 30_000, allowPreemption: false, defaultHostQuota: maxConcurrent,
|
|
94
|
+
managed: {
|
|
95
|
+
authorityRef: 'project', projectId, policyDigest, maxQueued: admission?.maxQueued ?? 128,
|
|
96
|
+
dimensionCaps: dimensions,
|
|
97
|
+
budgets: parallelism.budgets, retry: parallelism.retry,
|
|
98
|
+
installations: Object.fromEntries(Object.entries(parallelism.clis ?? {}).filter(([, cli]) => cli.enabled !== false)
|
|
99
|
+
.map(([id, cli]) => [id, { accountScope: cli.accountScope, quotaScope: cli.quotaScope }])),
|
|
100
|
+
members: Object.fromEntries(Object.entries(parallelism.groups ?? {}).flatMap(([group, definition]) => definition.members.filter(member => member.enabled !== false && parallelism.clis?.[member.cli]?.enabled !== false)
|
|
101
|
+
.map(member => [`${group}/${member.id}`, { cli: member.cli }]))),
|
|
102
|
+
},
|
|
103
|
+
};
|
|
104
|
+
}
|
|
105
|
+
/** Initialize a validated activation under the store lock. Policy replacement/migration
|
|
106
|
+
* is deliberately not supported here; active or uncertain work must first be reconciled. */
|
|
107
|
+
export function bindManagedAdmissionPolicy(store, policy) {
|
|
108
|
+
if (!policy.managed || policy.managed.authorityRef !== 'project')
|
|
109
|
+
throw new Error('managed project policy is required');
|
|
110
|
+
new SharedHostScheduler(store, policy);
|
|
111
|
+
store.transact(snapshot => {
|
|
112
|
+
if (snapshot.hostBinding || snapshot.migration || snapshot.authorityRedirect)
|
|
113
|
+
throw new Error('authority migration requires the trusted owner');
|
|
114
|
+
if (snapshot.policyBinding && canonical(snapshot.policyBinding) !== canonical(policy)) {
|
|
115
|
+
throw new Error('managed admission policy binding conflicts with the activated policy');
|
|
116
|
+
}
|
|
117
|
+
snapshot.policyBinding = structuredClone(policy);
|
|
118
|
+
});
|
|
119
|
+
}
|
|
14
120
|
export class InMemoryAdmissionStore {
|
|
15
121
|
snapshot = { revision: 0, records: {} };
|
|
16
122
|
transact(mutate) {
|
|
@@ -28,10 +134,45 @@ export class InMemoryAdmissionStore {
|
|
|
28
134
|
* concurrent writer receives a conflict and retries through its control loop. */
|
|
29
135
|
export class FileAdmissionStore {
|
|
30
136
|
path;
|
|
137
|
+
hostWritableRoots;
|
|
138
|
+
durableWrites = false;
|
|
31
139
|
constructor(path) {
|
|
32
140
|
this.path = path;
|
|
33
141
|
}
|
|
142
|
+
/** Migration checkpoints must survive a completed write on either authority. */
|
|
143
|
+
useDurableWrites() { this.durableWrites = true; }
|
|
144
|
+
/** Trusted executor bootstrap supplies the actual sandbox writable mounts.
|
|
145
|
+
* Mode checks do not isolate an unsandboxed process running as the same uid. */
|
|
146
|
+
protectHostAuthority(workerWritableRoots) {
|
|
147
|
+
if (!Array.isArray(workerWritableRoots) || workerWritableRoots.some(root => !isAbsolute(root)))
|
|
148
|
+
throw new Error('absolute worker writable roots are required');
|
|
149
|
+
this.hostWritableRoots = workerWritableRoots.map(root => realpathSync(root));
|
|
150
|
+
this.assertHostPath();
|
|
151
|
+
}
|
|
152
|
+
/** Only the trusted registration owner persists this protected location. */
|
|
153
|
+
hostAuthorityPath() {
|
|
154
|
+
if (!this.hostWritableRoots)
|
|
155
|
+
throw new Error('host authority path has not been protected');
|
|
156
|
+
this.readUnsafe();
|
|
157
|
+
return this.path;
|
|
158
|
+
}
|
|
159
|
+
assertHostPath() {
|
|
160
|
+
if (!this.hostWritableRoots)
|
|
161
|
+
return;
|
|
162
|
+
const parent = dirname(this.path);
|
|
163
|
+
if (!isAbsolute(this.path) || realpathSync(parent) !== resolve(parent))
|
|
164
|
+
throw new Error('host authority path must be canonical without symlink parents');
|
|
165
|
+
const directory = lstatSync(parent);
|
|
166
|
+
if (!directory.isDirectory() || (directory.mode & 0o077) !== 0 || directory.uid !== process.getuid?.())
|
|
167
|
+
throw new Error('host authority directory must be private to its owner');
|
|
168
|
+
for (const root of this.hostWritableRoots) {
|
|
169
|
+
const within = relative(root, resolve(this.path));
|
|
170
|
+
if (!within || (!within.startsWith('../') && within !== '..' && !isAbsolute(within)))
|
|
171
|
+
throw new Error('worker writable mount includes the host authority');
|
|
172
|
+
}
|
|
173
|
+
}
|
|
34
174
|
transact(mutate) {
|
|
175
|
+
this.assertHostPath();
|
|
35
176
|
mkdirSync(dirname(this.path), { recursive: true, mode: 0o700 });
|
|
36
177
|
const lockPath = `${this.path}.lock`;
|
|
37
178
|
let descriptor;
|
|
@@ -49,8 +190,28 @@ export class FileAdmissionStore {
|
|
|
49
190
|
const result = mutate(snapshot);
|
|
50
191
|
snapshot.revision += 1;
|
|
51
192
|
const temporary = `${this.path}.${process.pid}.tmp`;
|
|
52
|
-
|
|
193
|
+
if (this.hostWritableRoots || this.durableWrites) {
|
|
194
|
+
const file = openSync(temporary, 'wx', 0o600);
|
|
195
|
+
try {
|
|
196
|
+
writeFileSync(file, `${JSON.stringify(snapshot, null, 2)}\n`);
|
|
197
|
+
fsyncSync(file);
|
|
198
|
+
}
|
|
199
|
+
finally {
|
|
200
|
+
closeSync(file);
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
else
|
|
204
|
+
writeFileSync(temporary, `${JSON.stringify(snapshot, null, 2)}\n`, { mode: 0o600 });
|
|
53
205
|
renameSync(temporary, this.path);
|
|
206
|
+
if (this.hostWritableRoots || this.durableWrites) {
|
|
207
|
+
const directory = openSync(dirname(this.path), 'r');
|
|
208
|
+
try {
|
|
209
|
+
fsyncSync(directory);
|
|
210
|
+
}
|
|
211
|
+
finally {
|
|
212
|
+
closeSync(directory);
|
|
213
|
+
}
|
|
214
|
+
}
|
|
54
215
|
return result;
|
|
55
216
|
}
|
|
56
217
|
finally {
|
|
@@ -65,13 +226,28 @@ export class FileAdmissionStore {
|
|
|
65
226
|
return structuredClone(this.readUnsafe());
|
|
66
227
|
}
|
|
67
228
|
readUnsafe() {
|
|
229
|
+
this.assertHostPath();
|
|
230
|
+
if (this.hostWritableRoots || this.durableWrites) {
|
|
231
|
+
if (!this.hostWritableRoots && !existsSync(this.path))
|
|
232
|
+
return { revision: 0, records: {} };
|
|
233
|
+
const file = openSync(this.path, constants.O_RDONLY | constants.O_NOFOLLOW);
|
|
234
|
+
try {
|
|
235
|
+
const stat = fstatSync(file);
|
|
236
|
+
if (!stat.isFile() || stat.nlink !== 1 || (stat.mode & 0o077) !== 0 || stat.uid !== process.getuid?.() || stat.size > 16 * 1024 * 1024)
|
|
237
|
+
throw new Error('host authority must be a private bounded regular file');
|
|
238
|
+
return JSON.parse(readFileSync(file, 'utf8'));
|
|
239
|
+
}
|
|
240
|
+
finally {
|
|
241
|
+
closeSync(file);
|
|
242
|
+
}
|
|
243
|
+
}
|
|
68
244
|
if (!existsSync(this.path))
|
|
69
245
|
return { revision: 0, records: {} };
|
|
70
246
|
return JSON.parse(readFileSync(this.path, 'utf8'));
|
|
71
247
|
}
|
|
72
248
|
}
|
|
73
249
|
const terminalStates = new Set([
|
|
74
|
-
'denied', 'cancelled', 'timed-out', 'preempted',
|
|
250
|
+
'denied', 'cancelled', 'timed-out', 'preempted', 'settled',
|
|
75
251
|
]);
|
|
76
252
|
export class SharedHostScheduler {
|
|
77
253
|
store;
|
|
@@ -87,37 +263,110 @@ export class SharedHostScheduler {
|
|
|
87
263
|
if (policy.leaseTtlMs < 1 || policy.agingIntervalMs < 1) {
|
|
88
264
|
throw new Error('leaseTtlMs and agingIntervalMs must be positive');
|
|
89
265
|
}
|
|
266
|
+
if (policy.managed)
|
|
267
|
+
this.policy = structuredClone(policy);
|
|
268
|
+
const managed = this.policy.managed;
|
|
269
|
+
if (managed) {
|
|
270
|
+
managedDimensionKeys(managed.projectId, { kind: 'native' });
|
|
271
|
+
if (managed.authorityRef !== 'project' && !/^host:[a-z][a-z0-9-]{0,62}$/.test(managed.authorityRef))
|
|
272
|
+
throw new Error('invalid managed authority reference');
|
|
273
|
+
if (!/^sha256:[a-f0-9]{64}$/.test(managed.policyDigest))
|
|
274
|
+
throw new Error('invalid activated policy digest');
|
|
275
|
+
if (!Number.isSafeInteger(managed.maxQueued) || managed.maxQueued < 0)
|
|
276
|
+
throw new Error('maxQueued must be a nonnegative integer');
|
|
277
|
+
if (!Number.isSafeInteger(policy.maxConcurrent) || !Number.isFinite(policy.leaseTtlMs) || !Number.isFinite(policy.agingIntervalMs)) {
|
|
278
|
+
throw new Error('managed capacity and lease intervals must be finite');
|
|
279
|
+
}
|
|
280
|
+
for (const [key, cap] of Object.entries(managed.dimensionCaps)) {
|
|
281
|
+
if (!(managed.authorityRef === 'project' ? isCanonicalManagedDimension(managed.projectId, key) : isHostDimension(key)))
|
|
282
|
+
throw new Error('invalid canonical admission dimension key');
|
|
283
|
+
if (!Number.isSafeInteger(cap) || cap < 1)
|
|
284
|
+
throw new Error('dimension caps must be positive integers');
|
|
285
|
+
}
|
|
286
|
+
if (managed.budgets?.enforcement === 'hard')
|
|
287
|
+
throw new Error('OG_BUDGET_UNMEASURABLE: no enforcing spend adapter is bound');
|
|
288
|
+
validateBudgets(projectBudgets(managed.budgets?.perRun));
|
|
289
|
+
validateBudgets(projectBudgets(managed.budgets?.perTask));
|
|
290
|
+
for (const value of [managed.retry?.maxAttempts ?? 3, managed.retry?.baseDelaySeconds ?? 1, managed.retry?.maxDelaySeconds ?? 60]) {
|
|
291
|
+
if (!Number.isSafeInteger(value) || value < 1)
|
|
292
|
+
throw new Error('managed retry bounds must be positive integers');
|
|
293
|
+
}
|
|
294
|
+
if ((managed.retry?.baseDelaySeconds ?? 1) > (managed.retry?.maxDelaySeconds ?? 60))
|
|
295
|
+
throw new Error('invalid retry delay bounds');
|
|
296
|
+
if (managed.budgets?.reservationUsd !== undefined && (!Number.isFinite(managed.budgets.reservationUsd) || managed.budgets.reservationUsd <= 0))
|
|
297
|
+
throw new Error('reservationUsd must be positive');
|
|
298
|
+
const aliases = new Map();
|
|
299
|
+
for (const [cli, binding] of Object.entries(managed.installations)) {
|
|
300
|
+
managedDimensionKeys(managed.projectId, { kind: 'native', cli, quotaScope: binding.quotaScope });
|
|
301
|
+
if (!binding.accountScope?.trim() || !binding.quotaScope?.trim())
|
|
302
|
+
throw new Error('managed installations require account and quota bindings');
|
|
303
|
+
const quota = aliases.get(binding.accountScope);
|
|
304
|
+
if (quota !== undefined && quota !== binding.quotaScope)
|
|
305
|
+
throw new Error('ambiguous account aliases bind one account to different quota scopes');
|
|
306
|
+
aliases.set(binding.accountScope, binding.quotaScope);
|
|
307
|
+
}
|
|
308
|
+
}
|
|
90
309
|
}
|
|
91
310
|
submit(request) {
|
|
92
311
|
assertMissionWorkerHandoff(request, 'managed-admission');
|
|
93
312
|
this.validateRequest(request);
|
|
94
313
|
return this.store.transact(snapshot => {
|
|
95
|
-
|
|
314
|
+
this.assertPolicyBinding(snapshot);
|
|
315
|
+
this.assertSubmissionsOpen(snapshot);
|
|
316
|
+
const requestKey = this.requestKey(request.requestId);
|
|
317
|
+
const existing = snapshot.records[requestKey];
|
|
96
318
|
if (existing) {
|
|
97
|
-
if (!sameRequest(existing, request)) {
|
|
319
|
+
if (!(this.policy.managed ? sameManagedRequest(existing, request) : sameRequest(existing, request))) {
|
|
98
320
|
throw new Error(`request '${request.requestId}' conflicts with an existing admission`);
|
|
99
321
|
}
|
|
100
|
-
return existing;
|
|
322
|
+
return this.policy.managed ? structuredClone(existing) : existing;
|
|
101
323
|
}
|
|
102
|
-
|
|
324
|
+
if (this.policy.managed)
|
|
325
|
+
this.reconcile(snapshot);
|
|
326
|
+
snapshot.records[requestKey] = {
|
|
103
327
|
...structuredClone(request),
|
|
328
|
+
...(snapshot.hostBinding ? { projectId: this.policy.managed.projectId } : {}),
|
|
104
329
|
state: 'queued',
|
|
105
330
|
revision: 1,
|
|
106
331
|
reason: 'awaiting shared-host capacity',
|
|
332
|
+
...(this.policy.managed ? {
|
|
333
|
+
attemptGeneration: 1,
|
|
334
|
+
leaseGeneration: 1,
|
|
335
|
+
dimensions: this.reservationDimensions(request),
|
|
336
|
+
reservedSlots: request.managed?.lowerBound ?? 1,
|
|
337
|
+
accounting: this.bindAllowance(snapshot, request),
|
|
338
|
+
} : {}),
|
|
107
339
|
};
|
|
108
340
|
this.reconcile(snapshot);
|
|
109
|
-
|
|
341
|
+
if (this.policy.managed && snapshot.records[requestKey].state === 'queued'
|
|
342
|
+
&& (Object.values(snapshot.records).filter(record => record.state === 'queued' && (!snapshot.hostBinding || record.projectId === this.policy.managed.projectId)).length > this.policy.managed.maxQueued
|
|
343
|
+
|| (snapshot.hostBinding && Object.values(snapshot.records).filter(record => record.state === 'queued').length > snapshot.hostBinding.maxQueued))) {
|
|
344
|
+
const record = snapshot.records[requestKey];
|
|
345
|
+
record.state = 'denied';
|
|
346
|
+
record.reason = 'managed admission queue is full';
|
|
347
|
+
record.finishedAt = new Date(this.clock()).toISOString();
|
|
348
|
+
record.revision += 1;
|
|
349
|
+
}
|
|
350
|
+
return structuredClone(snapshot.records[requestKey]);
|
|
110
351
|
});
|
|
111
352
|
}
|
|
112
353
|
reconcileNow() {
|
|
113
354
|
return this.store.transact(snapshot => {
|
|
355
|
+
this.assertPolicyBinding(snapshot);
|
|
114
356
|
this.reconcile(snapshot);
|
|
115
|
-
return
|
|
357
|
+
return this.projectSnapshot(snapshot);
|
|
116
358
|
});
|
|
117
359
|
}
|
|
118
|
-
renew(requestId) {
|
|
360
|
+
renew(requestId, generation) {
|
|
119
361
|
return this.store.transact(snapshot => {
|
|
362
|
+
this.assertPolicyBinding(snapshot);
|
|
120
363
|
const record = this.required(snapshot, requestId);
|
|
364
|
+
if (this.policy.managed) {
|
|
365
|
+
this.assertGeneration(record, generation);
|
|
366
|
+
this.reconcile(snapshot);
|
|
367
|
+
if (record.state === 'uncertain')
|
|
368
|
+
return structuredClone(record);
|
|
369
|
+
}
|
|
121
370
|
if (record.state !== 'admitted')
|
|
122
371
|
throw new Error(`cannot renew ${record.state} admission`);
|
|
123
372
|
record.leaseExpiresAt = new Date(this.clock() + this.policy.leaseTtlMs).toISOString();
|
|
@@ -128,36 +377,247 @@ export class SharedHostScheduler {
|
|
|
128
377
|
}
|
|
129
378
|
release(requestId) {
|
|
130
379
|
return this.store.transact(snapshot => {
|
|
380
|
+
this.assertPolicyBinding(snapshot);
|
|
131
381
|
const record = this.required(snapshot, requestId);
|
|
382
|
+
if (this.policy.managed) {
|
|
383
|
+
throw new Error('managed capacity requires proof-checked settlement');
|
|
384
|
+
}
|
|
132
385
|
delete snapshot.records[requestId];
|
|
133
386
|
if (record.state === 'admitted')
|
|
134
387
|
this.reconcile(snapshot);
|
|
135
388
|
return structuredClone(snapshot);
|
|
136
389
|
});
|
|
137
390
|
}
|
|
138
|
-
cancel(requestId) {
|
|
391
|
+
cancel(requestId, generation) {
|
|
139
392
|
return this.store.transact(snapshot => {
|
|
393
|
+
this.assertPolicyBinding(snapshot);
|
|
140
394
|
const record = this.required(snapshot, requestId);
|
|
395
|
+
if (this.policy.managed)
|
|
396
|
+
this.assertGeneration(record, generation);
|
|
141
397
|
if (terminalStates.has(record.state))
|
|
142
398
|
return structuredClone(record);
|
|
143
|
-
record.state = 'cancelled';
|
|
399
|
+
record.state = this.policy.managed && (record.state === 'admitted' || record.state === 'uncertain') ? 'uncertain' : 'cancelled';
|
|
144
400
|
record.reason = 'cancelled by orchestrator';
|
|
145
|
-
record.
|
|
401
|
+
if (record.state !== 'uncertain')
|
|
402
|
+
record.finishedAt = new Date(this.clock()).toISOString();
|
|
403
|
+
record.revision += 1;
|
|
404
|
+
this.reconcile(snapshot);
|
|
405
|
+
return structuredClone(record);
|
|
406
|
+
});
|
|
407
|
+
}
|
|
408
|
+
/** Persist before spawning. A crash on either side requires reconciliation. */
|
|
409
|
+
claimLaunch(requestId, generation) {
|
|
410
|
+
return this.store.transact(snapshot => {
|
|
411
|
+
this.assertPolicyBinding(snapshot);
|
|
412
|
+
this.assertSubmissionsOpen(snapshot);
|
|
413
|
+
const record = this.required(snapshot, requestId);
|
|
414
|
+
this.assertGeneration(record, generation);
|
|
415
|
+
if (record.state !== 'admitted' || Date.parse(record.leaseExpiresAt) <= this.clock())
|
|
416
|
+
throw new Error('launch requires a current admitted lease');
|
|
417
|
+
if (!record.accounting || [record.accounting.runKey, record.accounting.taskKey].some(key => Date.parse(snapshot.allowances?.[key]?.deadlineAt ?? '') <= this.clock()))
|
|
418
|
+
throw new Error('cumulative allowance deadline elapsed before launch');
|
|
419
|
+
if (record.launchClaimedAt)
|
|
420
|
+
throw new Error('launch already claimed; reconcile before dispatch');
|
|
421
|
+
record.launchClaimedAt = new Date(this.clock()).toISOString();
|
|
422
|
+
record.revision += 1;
|
|
423
|
+
return structuredClone(record);
|
|
424
|
+
});
|
|
425
|
+
}
|
|
426
|
+
bindExecution(requestId, generation, execution) {
|
|
427
|
+
if (!Number.isSafeInteger(execution.pid) || execution.pid <= 0 || typeof execution.startedAt !== 'string' || !execution.startedAt.trim()) {
|
|
428
|
+
throw new Error('execution requires a PID and OS process birth identity');
|
|
429
|
+
}
|
|
430
|
+
return this.store.transact(snapshot => {
|
|
431
|
+
this.assertPolicyBinding(snapshot);
|
|
432
|
+
const record = this.required(snapshot, requestId);
|
|
433
|
+
this.assertGeneration(record, generation);
|
|
434
|
+
if (!record.launchClaimedAt || !['admitted', 'uncertain'].includes(record.state))
|
|
435
|
+
throw new Error('execution requires a claimed launch');
|
|
436
|
+
if (record.execution) {
|
|
437
|
+
if (canonical(record.execution) !== canonical(execution))
|
|
438
|
+
throw new Error('execution identity conflicts; possible PID reuse');
|
|
439
|
+
return structuredClone(record);
|
|
440
|
+
}
|
|
441
|
+
record.execution = structuredClone(execution);
|
|
442
|
+
record.revision += 1;
|
|
443
|
+
return structuredClone(record);
|
|
444
|
+
});
|
|
445
|
+
}
|
|
446
|
+
stop(requestId, generation, reason = 'cancel') {
|
|
447
|
+
if (!['cancel', 'disconnect', 'preempt', 'observer-loss'].includes(reason))
|
|
448
|
+
throw new Error('invalid stop reason');
|
|
449
|
+
return this.store.transact(snapshot => {
|
|
450
|
+
this.assertPolicyBinding(snapshot);
|
|
451
|
+
const record = this.required(snapshot, requestId);
|
|
452
|
+
this.assertGeneration(record, generation);
|
|
453
|
+
if (terminalStates.has(record.state))
|
|
454
|
+
return structuredClone(record);
|
|
455
|
+
record.state = record.state === 'queued' ? 'cancelled' : 'uncertain';
|
|
456
|
+
record.reason = `${reason}; termination is uncertain`;
|
|
457
|
+
if (record.state === 'cancelled')
|
|
458
|
+
record.finishedAt = new Date(this.clock()).toISOString();
|
|
459
|
+
record.revision += 1;
|
|
460
|
+
this.reconcile(snapshot);
|
|
461
|
+
return structuredClone(record);
|
|
462
|
+
});
|
|
463
|
+
}
|
|
464
|
+
disconnect(requestId, generation) {
|
|
465
|
+
return this.stop(requestId, generation, 'disconnect');
|
|
466
|
+
}
|
|
467
|
+
/** Supervision observations may advance progress, but never release capacity. */
|
|
468
|
+
observeHealth(requestId, generation, health) {
|
|
469
|
+
assertValidWorkerPayload('workerHealthSummary', health);
|
|
470
|
+
return this.store.transact(snapshot => {
|
|
471
|
+
this.assertPolicyBinding(snapshot);
|
|
472
|
+
const record = this.required(snapshot, requestId);
|
|
473
|
+
this.assertGeneration(record, generation);
|
|
474
|
+
if (health.observerGeneration !== generation)
|
|
475
|
+
throw new Error('stale health observer generation');
|
|
476
|
+
if (!record.accounting?.attemptNumber)
|
|
477
|
+
throw new Error('health requires an admitted attempt');
|
|
478
|
+
if (record.state === 'settled')
|
|
479
|
+
return structuredClone(record);
|
|
480
|
+
this.reconcile(snapshot);
|
|
481
|
+
const verdict = this.healthVerdict(record, generation, health);
|
|
482
|
+
const trace = projectHealthTrace(health, record.health?.state, { lastProgressAt: record.lastProgressAt ?? null });
|
|
483
|
+
record.lastProgressAt = advanceProgressWatermark({ lastProgressAt: record.lastProgressAt ?? null }, trace.progressCredit, health).lastProgressAt;
|
|
484
|
+
if (!record.health || Date.parse(health.lastObservedAt ?? '') >= Date.parse(record.health.lastObservedAt ?? '') || record.health.lastObservedAt === null) {
|
|
485
|
+
record.health = structuredClone(health);
|
|
486
|
+
}
|
|
487
|
+
if (verdict === 'retain-uncertain') {
|
|
488
|
+
record.state = 'uncertain';
|
|
489
|
+
record.reason = 'supervisor observation is uncertain; reservation retained';
|
|
490
|
+
}
|
|
491
|
+
record.revision += 1;
|
|
492
|
+
return structuredClone(record);
|
|
493
|
+
});
|
|
494
|
+
}
|
|
495
|
+
/** Ingest final evidence without conflating it with a supervisor stop proof. */
|
|
496
|
+
acceptFinal(requestId, generation, health, usage) {
|
|
497
|
+
assertValidWorkerPayload('workerHealthSummary', health);
|
|
498
|
+
if (usage)
|
|
499
|
+
assertValidWorkerPayload('workerUsage', usage);
|
|
500
|
+
return this.store.transact(snapshot => {
|
|
501
|
+
this.assertPolicyBinding(snapshot);
|
|
502
|
+
const record = this.required(snapshot, requestId);
|
|
503
|
+
this.assertGeneration(record, generation);
|
|
504
|
+
if (health.observerGeneration !== generation)
|
|
505
|
+
throw new Error('stale final observer generation');
|
|
506
|
+
if (!record.accounting?.attemptNumber)
|
|
507
|
+
throw new Error('final requires an admitted attempt');
|
|
508
|
+
const evidence = { health, ...(usage ? { usage } : {}) };
|
|
509
|
+
if (record.finalEvidence) {
|
|
510
|
+
if (canonical(record.finalEvidence) !== canonical(evidence))
|
|
511
|
+
throw new Error('conflicting final evidence');
|
|
512
|
+
return structuredClone(record);
|
|
513
|
+
}
|
|
514
|
+
this.reconcile(snapshot);
|
|
515
|
+
record.finalEvidence = structuredClone(evidence);
|
|
516
|
+
record.revision += 1;
|
|
517
|
+
return structuredClone(record);
|
|
518
|
+
});
|
|
519
|
+
}
|
|
520
|
+
assertGeneration(record, generation) {
|
|
521
|
+
if (!this.policy.managed)
|
|
522
|
+
throw new Error('operation requires managed admission');
|
|
523
|
+
if (!Number.isSafeInteger(generation) || generation < 1 || generation !== record.leaseGeneration)
|
|
524
|
+
throw new Error('stale admission generation');
|
|
525
|
+
}
|
|
526
|
+
healthVerdict(record, generation, health) {
|
|
527
|
+
return decideLeaseMapping({ leaseGeneration: record.leaseGeneration, observedGeneration: generation,
|
|
528
|
+
leaseState: record.state === 'uncertain' ? 'cancelled-disconnected' : 'admitted',
|
|
529
|
+
settled: record.state === 'settled', health });
|
|
530
|
+
}
|
|
531
|
+
/** Called with trusted supervisor observations, never worker claims/status strings. */
|
|
532
|
+
settle(requestId, generation, health, usage, failure, execution) {
|
|
533
|
+
assertValidWorkerPayload('workerUsage', usage);
|
|
534
|
+
return this.store.transact(snapshot => {
|
|
535
|
+
this.assertPolicyBinding(snapshot);
|
|
536
|
+
if (!this.policy.managed)
|
|
537
|
+
throw new Error('settlement requires managed admission');
|
|
538
|
+
const record = this.required(snapshot, requestId);
|
|
539
|
+
this.assertGeneration(record, generation);
|
|
540
|
+
const accounting = record.accounting;
|
|
541
|
+
if (!accounting?.attemptNumber)
|
|
542
|
+
throw new Error('cannot settle an unlaunched attempt');
|
|
543
|
+
let verdict = this.healthVerdict(record, generation, health);
|
|
544
|
+
if (verdict === 'deny-stale')
|
|
545
|
+
throw new Error('stale settlement generation');
|
|
546
|
+
if (record.execution && canonical(record.execution) !== canonical(execution))
|
|
547
|
+
throw new Error('stop proof execution identity mismatch; possible PID reuse');
|
|
548
|
+
if (verdict === 'release-once' && record.launchClaimedAt && !record.execution)
|
|
549
|
+
verdict = 'retain-uncertain';
|
|
550
|
+
if (verdict === 'deny-already-settled') {
|
|
551
|
+
if (canonical(accounting.usage) !== canonical(usage))
|
|
552
|
+
throw new Error('conflicting settled usage');
|
|
553
|
+
return structuredClone(record);
|
|
554
|
+
}
|
|
555
|
+
if (failure)
|
|
556
|
+
this.applyFailure(snapshot, record, failure);
|
|
557
|
+
// An unconfirmed observation cannot erase a previous measured charge.
|
|
558
|
+
if (accounting.usage && !canRefineUsage(accounting.usage, usage))
|
|
559
|
+
throw new Error('conflicting attempt usage');
|
|
560
|
+
accounting.usage = structuredClone(usage);
|
|
561
|
+
if (verdict === 'release-once') {
|
|
562
|
+
record.state = 'settled';
|
|
563
|
+
record.finishedAt = new Date(this.clock()).toISOString();
|
|
564
|
+
accounting.stopEvidence = structuredClone(health);
|
|
565
|
+
record.reason = 'current-generation termination proof reconciled';
|
|
566
|
+
}
|
|
567
|
+
else {
|
|
568
|
+
record.state = 'uncertain';
|
|
569
|
+
record.reason = 'termination proof unavailable or not current; reservation retained';
|
|
570
|
+
}
|
|
146
571
|
record.revision += 1;
|
|
147
572
|
this.reconcile(snapshot);
|
|
148
573
|
return structuredClone(record);
|
|
149
574
|
});
|
|
150
575
|
}
|
|
576
|
+
/** A 429 observation cools the authority-bound account even if stop is uncertain. */
|
|
577
|
+
recordRateLimit(requestId, generation, failure = {}) {
|
|
578
|
+
return this.recordFailure(requestId, generation, { class: 'rate-limit', retryable: true, ...failure });
|
|
579
|
+
}
|
|
580
|
+
recordFailure(requestId, generation, failure) {
|
|
581
|
+
return this.store.transact(snapshot => {
|
|
582
|
+
this.assertPolicyBinding(snapshot);
|
|
583
|
+
if (!this.policy.managed)
|
|
584
|
+
throw new Error('failure observation requires managed admission');
|
|
585
|
+
const record = this.required(snapshot, requestId);
|
|
586
|
+
if (record.leaseGeneration !== generation)
|
|
587
|
+
throw new Error('stale failure observation generation');
|
|
588
|
+
if (!record.accounting?.attemptNumber)
|
|
589
|
+
throw new Error('failure observation requires an admitted attempt');
|
|
590
|
+
this.applyFailure(snapshot, record, failure);
|
|
591
|
+
return structuredClone(record);
|
|
592
|
+
});
|
|
593
|
+
}
|
|
151
594
|
snapshot() {
|
|
152
|
-
|
|
595
|
+
const snapshot = this.store.read();
|
|
596
|
+
if (this.policy.managed?.authorityRef.startsWith('host:'))
|
|
597
|
+
this.assertPolicyBinding(snapshot);
|
|
598
|
+
return this.projectSnapshot(snapshot);
|
|
599
|
+
}
|
|
600
|
+
projectSnapshot(value) {
|
|
601
|
+
const snapshot = structuredClone(value);
|
|
602
|
+
if (this.policy.managed?.authorityRef.startsWith('host:')) {
|
|
603
|
+
const projectId = this.policy.managed.projectId;
|
|
604
|
+
snapshot.records = Object.fromEntries(Object.values(snapshot.records)
|
|
605
|
+
.filter(record => record.projectId === projectId).map(record => [record.requestId, record]));
|
|
606
|
+
snapshot.allowances = Object.fromEntries(Object.entries(snapshot.allowances ?? {})
|
|
607
|
+
.filter(([key]) => JSON.parse(key)[0] === projectId));
|
|
608
|
+
}
|
|
609
|
+
return snapshot;
|
|
153
610
|
}
|
|
154
611
|
reconcile(snapshot) {
|
|
155
612
|
const now = this.clock();
|
|
156
613
|
for (const record of Object.values(snapshot.records)) {
|
|
614
|
+
if (this.policy.managed && record.managed && !record.accounting)
|
|
615
|
+
throw new Error('managed allowance migration required; reconcile pre-allowance reservations before dispatch');
|
|
157
616
|
if (record.state === 'admitted' && Date.parse(record.leaseExpiresAt ?? '') <= now) {
|
|
158
|
-
record.state = 'timed-out';
|
|
159
|
-
record.reason = 'admission lease expired; capacity recovered';
|
|
160
|
-
|
|
617
|
+
record.state = this.policy.managed ? 'uncertain' : 'timed-out';
|
|
618
|
+
record.reason = this.policy.managed ? 'admission lease expired; termination is uncertain' : 'admission lease expired; capacity recovered';
|
|
619
|
+
if (!this.policy.managed)
|
|
620
|
+
record.finishedAt = new Date(now).toISOString();
|
|
161
621
|
record.revision += 1;
|
|
162
622
|
}
|
|
163
623
|
else if (record.state === 'queued' && Date.parse(record.submittedAt) + record.queueTimeoutMs <= now) {
|
|
@@ -167,24 +627,154 @@ export class SharedHostScheduler {
|
|
|
167
627
|
record.revision += 1;
|
|
168
628
|
}
|
|
169
629
|
}
|
|
630
|
+
if (snapshot.migration || snapshot.authorityRedirect)
|
|
631
|
+
return;
|
|
170
632
|
let queued = Object.values(snapshot.records)
|
|
171
633
|
.filter(record => record.state === 'queued')
|
|
172
634
|
.sort((a, b) => this.compare(a, b, now));
|
|
173
635
|
for (const candidate of queued) {
|
|
636
|
+
if (this.policy.managed && !this.allowAttempt(snapshot, candidate, now))
|
|
637
|
+
continue;
|
|
174
638
|
if (!this.hasCapacity(snapshot, candidate)) {
|
|
175
|
-
if (this.policy.allowPreemption)
|
|
639
|
+
if (this.policy.allowPreemption && !this.policy.managed)
|
|
176
640
|
this.tryPreempt(snapshot, candidate, now);
|
|
177
641
|
}
|
|
178
642
|
if (!this.hasCapacity(snapshot, candidate))
|
|
179
643
|
continue;
|
|
644
|
+
if (candidate.accounting) {
|
|
645
|
+
candidate.accounting.attemptNumber = Object.values(snapshot.records).filter(record => record.accounting?.taskKey === candidate.accounting.taskKey && record.accounting.attemptNumber !== undefined)
|
|
646
|
+
.reduce((sum, record) => sum + record.accounting.attemptCount, 0) + 1;
|
|
647
|
+
}
|
|
180
648
|
candidate.state = 'admitted';
|
|
181
649
|
candidate.reason = 'admitted by shared-host policy';
|
|
182
650
|
candidate.admittedAt = new Date(now).toISOString();
|
|
183
|
-
candidate.leaseExpiresAt = new Date(now + this.
|
|
651
|
+
candidate.leaseExpiresAt = new Date(now + this.policyFor(snapshot, candidate).leaseTtlMs).toISOString();
|
|
184
652
|
candidate.revision += 1;
|
|
185
653
|
}
|
|
186
654
|
queued = [];
|
|
187
655
|
}
|
|
656
|
+
applyFailure(snapshot, record, failure) {
|
|
657
|
+
const accounting = record.accounting;
|
|
658
|
+
if (!['rate-limit', 'unavailable', 'timeout', 'auth', 'invalid-request', 'policy', 'parse', 'verification', 'unknown'].includes(failure.class)
|
|
659
|
+
|| typeof failure.retryable !== 'boolean' || (failure.resetAt !== undefined && (typeof failure.resetAt !== 'string' || failure.resetAt.length > 128))) {
|
|
660
|
+
throw new Error('invalid attempt failure');
|
|
661
|
+
}
|
|
662
|
+
if (accounting.failure) {
|
|
663
|
+
if (canonical(accounting.failure) !== canonical(failure))
|
|
664
|
+
throw new Error('conflicting failure observation');
|
|
665
|
+
return;
|
|
666
|
+
}
|
|
667
|
+
const retry = this.policy.managed.retry;
|
|
668
|
+
const cooldowns = snapshot.cooldowns ??= {};
|
|
669
|
+
const quota = record.managed?.cli ? (this.policy.managed.upstream ?? this.policy.managed.installations)[record.managed.cli].quotaScope : null;
|
|
670
|
+
if (failure.class === 'rate-limit' && !quota)
|
|
671
|
+
throw new Error('quota observation requires an admitted installation');
|
|
672
|
+
const key = `quota/${encodeURIComponent(quota ?? '')}`;
|
|
673
|
+
const failures = failure.class === 'rate-limit' ? (cooldowns[key]?.failures ?? 0) + 1
|
|
674
|
+
: Object.values(snapshot.records).filter(previous => previous.accounting?.taskKey === accounting.taskKey && previous.accounting.failure).length + 1;
|
|
675
|
+
const delay = Math.min(retry?.maxDelaySeconds ?? 60, (retry?.baseDelaySeconds ?? 1) * 2 ** Math.min(failures - 1, 30)) * 1000;
|
|
676
|
+
const now = this.clock();
|
|
677
|
+
const reset = Date.parse(failure.resetAt ?? '');
|
|
678
|
+
const until = Math.max(now + delay, Number.isFinite(reset) ? reset : now, failure.class === 'rate-limit' ? Date.parse(cooldowns[key]?.until ?? '') || 0 : 0);
|
|
679
|
+
if (!Number.isFinite(new Date(until).getTime()))
|
|
680
|
+
throw new Error('retry backoff exceeds timestamp range');
|
|
681
|
+
if (failure.class === 'rate-limit')
|
|
682
|
+
cooldowns[key] = { until: new Date(until).toISOString(), failures };
|
|
683
|
+
accounting.notBefore = new Date(until).toISOString();
|
|
684
|
+
accounting.failure = structuredClone(failure);
|
|
685
|
+
record.revision += 1;
|
|
686
|
+
}
|
|
687
|
+
bindAllowance(snapshot, request) {
|
|
688
|
+
const policy = this.policy.managed;
|
|
689
|
+
const allowance = request.allowance ?? { runId: request.orchestratorId, taskId: request.metadata?.missionId ?? request.requestId };
|
|
690
|
+
const runKey = this.allowanceKey(JSON.stringify(['run', allowance.runId]));
|
|
691
|
+
const taskKey = this.allowanceKey(JSON.stringify(['task', allowance.runId, allowance.taskId]));
|
|
692
|
+
const scopes = snapshot.allowances ??= {};
|
|
693
|
+
const bind = (key, limits) => {
|
|
694
|
+
const previous = scopes[key];
|
|
695
|
+
const effective = intersectBudgets(previous?.limits, limits);
|
|
696
|
+
const startedAt = previous?.startedAt ?? new Date(this.clock()).toISOString();
|
|
697
|
+
const deadline = effective.timeoutSeconds === undefined ? undefined : Date.parse(startedAt) + effective.timeoutSeconds * 1000;
|
|
698
|
+
if (deadline !== undefined && !Number.isFinite(new Date(deadline).getTime()))
|
|
699
|
+
throw new Error('allowance deadline exceeds timestamp range');
|
|
700
|
+
return scopes[key] = { limits: effective, startedAt,
|
|
701
|
+
...(deadline === undefined ? {} : { deadlineAt: new Date(deadline).toISOString() }) };
|
|
702
|
+
};
|
|
703
|
+
const run = bind(runKey, intersectBudgets(projectBudgets(policy.budgets?.perRun), allowance.runBudgets));
|
|
704
|
+
const task = bind(taskKey, intersectBudgets({ maxAttempts: policy.retry?.maxAttempts ?? 3 }, projectBudgets(policy.budgets?.perTask), allowance.taskBudgets));
|
|
705
|
+
const limits = intersectBudgets(run.limits, task.limits);
|
|
706
|
+
// Missing observations reserve the entire bounded envelope unless the owner
|
|
707
|
+
// supplied a smaller estimate; unbounded dimensions remain explicitly null.
|
|
708
|
+
const reservation = allowance.reservation ?? {
|
|
709
|
+
tokens: limits.maxTokens ?? null,
|
|
710
|
+
costUsd: policy.budgets?.reservationUsd ?? limits.maxCostUsd ?? null,
|
|
711
|
+
};
|
|
712
|
+
return { runKey, taskKey, reservation: structuredClone(reservation), usage: null,
|
|
713
|
+
attemptCount: Math.min(allowance.attempts ?? 1, limits.maxAttempts ?? Number.MAX_SAFE_INTEGER),
|
|
714
|
+
};
|
|
715
|
+
}
|
|
716
|
+
allowAttempt(snapshot, candidate, now) {
|
|
717
|
+
const accounting = candidate.accounting;
|
|
718
|
+
const deny = (reason) => {
|
|
719
|
+
candidate.state = 'denied';
|
|
720
|
+
candidate.reason = reason;
|
|
721
|
+
candidate.finishedAt = new Date(now).toISOString();
|
|
722
|
+
candidate.revision += 1;
|
|
723
|
+
return false;
|
|
724
|
+
};
|
|
725
|
+
const attempts = Object.values(snapshot.records).filter(record => record.accounting?.attemptNumber !== undefined);
|
|
726
|
+
const taskAttempts = attempts.filter(record => record.accounting.taskKey === accounting.taskKey);
|
|
727
|
+
if (taskAttempts.length) {
|
|
728
|
+
const previous = taskAttempts.sort((a, b) => a.accounting.attemptNumber - b.accounting.attemptNumber).at(-1);
|
|
729
|
+
if (candidate.allowance?.previousRequestId !== previous.requestId)
|
|
730
|
+
return deny('new attempt must reference the latest task attempt');
|
|
731
|
+
if (previous.state !== 'settled')
|
|
732
|
+
return deny('previous attempt requires proof-checked reconciliation before replacement');
|
|
733
|
+
const failure = previous.accounting.failure;
|
|
734
|
+
const purpose = candidate.allowance?.purpose ?? 'retry';
|
|
735
|
+
if (purpose === 'retry' || purpose === 'fallback' || failure?.class === 'rate-limit') {
|
|
736
|
+
if (!failure || !failure.retryable || !['rate-limit', 'unavailable', 'timeout'].includes(failure.class))
|
|
737
|
+
return deny('previous failure does not permit automatic retry');
|
|
738
|
+
if (!(this.policyFor(snapshot, candidate).managed.retry?.on ?? ['rate-limit', 'unavailable', 'timeout']).includes(failure.class))
|
|
739
|
+
return deny(`${failure.class} retries are disabled`);
|
|
740
|
+
}
|
|
741
|
+
}
|
|
742
|
+
else if (candidate.allowance?.previousRequestId)
|
|
743
|
+
return deny('unknown previous task attempt');
|
|
744
|
+
for (const key of [accounting.runKey, accounting.taskKey]) {
|
|
745
|
+
const scope = snapshot.allowances[key];
|
|
746
|
+
if (Date.parse(scope.deadlineAt ?? '') <= now)
|
|
747
|
+
return deny('cumulative allowance deadline elapsed');
|
|
748
|
+
const prior = attempts.filter(record => record.accounting.runKey === key || record.accounting.taskKey === key);
|
|
749
|
+
if (scope.limits.maxAttempts !== undefined && prior.reduce((sum, record) => sum + record.accounting.attemptCount, 0) + accounting.attemptCount > scope.limits.maxAttempts)
|
|
750
|
+
return deny('cumulative attempt allowance exhausted');
|
|
751
|
+
for (const [dimension, limit] of [['tokens', scope.limits.maxTokens], ['costUsd', scope.limits.maxCostUsd]]) {
|
|
752
|
+
if (limit === undefined)
|
|
753
|
+
continue;
|
|
754
|
+
let used = 0;
|
|
755
|
+
for (const record of prior) {
|
|
756
|
+
const value = chargedUsage(record, dimension);
|
|
757
|
+
if (value === null)
|
|
758
|
+
return deny(`unknown ${dimension} charge has no finite reservation`);
|
|
759
|
+
used += value;
|
|
760
|
+
}
|
|
761
|
+
const reserved = accounting.reservation[dimension];
|
|
762
|
+
if (reserved === null || reserved === 0 || used + reserved > limit)
|
|
763
|
+
return deny(`cumulative ${dimension} reservation exceeds allowance`);
|
|
764
|
+
}
|
|
765
|
+
}
|
|
766
|
+
if (taskAttempts.some(record => Date.parse(record.accounting.notBefore ?? '') > now))
|
|
767
|
+
return false;
|
|
768
|
+
if (this.policyFor(snapshot, candidate).managed.budgets?.unknownCost === 'deny' && accounting.reservation.costUsd === null)
|
|
769
|
+
return deny('unknown cost is denied by policy');
|
|
770
|
+
const quotaScope = candidate.managed?.cli && snapshot.hostBinding
|
|
771
|
+
? this.policyFor(snapshot, candidate).managed.upstream[candidate.managed.cli].quotaScope : candidate.managed?.quotaScope;
|
|
772
|
+
const quotas = quotaScope ? [`quota/${encodeURIComponent(quotaScope)}`]
|
|
773
|
+
: Object.keys(snapshot.cooldowns ?? {});
|
|
774
|
+
if (quotas.some(key => Date.parse(snapshot.cooldowns?.[key]?.until ?? '') > now))
|
|
775
|
+
return false;
|
|
776
|
+
return true;
|
|
777
|
+
}
|
|
188
778
|
tryPreempt(snapshot, candidate, now) {
|
|
189
779
|
const victims = Object.values(snapshot.records)
|
|
190
780
|
.filter(record => record.state === 'admitted' && record.preemptible === true)
|
|
@@ -207,18 +797,31 @@ export class SharedHostScheduler {
|
|
|
207
797
|
victim.revision += 1;
|
|
208
798
|
}
|
|
209
799
|
hasCapacity(snapshot, candidate) {
|
|
210
|
-
const
|
|
211
|
-
|
|
800
|
+
const policy = this.policyFor(snapshot, candidate);
|
|
801
|
+
const active = Object.values(snapshot.records).filter(record => record.state === 'admitted' || (policy.managed && record.state === 'uncertain'));
|
|
802
|
+
const slots = policy.managed ? candidate.reservedSlots ?? 1 : 1;
|
|
803
|
+
if (active.reduce((sum, record) => sum + (policy.managed ? record.reservedSlots ?? 1 : 1), 0) + slots > policy.maxConcurrent)
|
|
212
804
|
return false;
|
|
213
|
-
if (
|
|
805
|
+
if (policy.managed) {
|
|
806
|
+
for (const key of candidate.dimensions ?? []) {
|
|
807
|
+
const cap = policy.managed.dimensionCaps[key];
|
|
808
|
+
if (cap === undefined)
|
|
809
|
+
continue;
|
|
810
|
+
const used = active.filter(record => !record.dimensions || record.dimensions.includes(key))
|
|
811
|
+
.reduce((sum, record) => sum + (record.reservedSlots ?? 1), 0);
|
|
812
|
+
if (used + slots > cap)
|
|
813
|
+
return false;
|
|
814
|
+
}
|
|
815
|
+
}
|
|
816
|
+
if (!belowQuota(active, 'environment', candidate.environment, policy.environmentQuotas, slots, !!policy.managed))
|
|
214
817
|
return false;
|
|
215
|
-
if (!belowQuota(active, 'provider', candidate.provider,
|
|
818
|
+
if (!belowQuota(active, 'provider', candidate.provider, policy.providerQuotas, slots, !!policy.managed))
|
|
216
819
|
return false;
|
|
217
820
|
const runtimeQuotas = {
|
|
218
|
-
host:
|
|
219
|
-
...
|
|
821
|
+
host: policy.defaultHostQuota ?? 1,
|
|
822
|
+
...policy.runtimeQuotas,
|
|
220
823
|
};
|
|
221
|
-
return belowQuota(active, 'runtimeKind', candidate.runtimeKind, runtimeQuotas);
|
|
824
|
+
return belowQuota(active, 'runtimeKind', candidate.runtimeKind, runtimeQuotas, slots, !!policy.managed);
|
|
222
825
|
}
|
|
223
826
|
effectivePriority(record, now) {
|
|
224
827
|
const waited = Math.max(0, now - Date.parse(record.submittedAt));
|
|
@@ -230,7 +833,7 @@ export class SharedHostScheduler {
|
|
|
230
833
|
|| a.requestId.localeCompare(b.requestId);
|
|
231
834
|
}
|
|
232
835
|
required(snapshot, requestId) {
|
|
233
|
-
const record = snapshot.records[requestId];
|
|
836
|
+
const record = snapshot.records[this.requestKey(requestId)];
|
|
234
837
|
if (!record)
|
|
235
838
|
throw new Error(`unknown admission request '${requestId}'`);
|
|
236
839
|
return record;
|
|
@@ -245,13 +848,437 @@ export class SharedHostScheduler {
|
|
|
245
848
|
throw new Error('submittedAt must be a timestamp');
|
|
246
849
|
if (request.queueTimeoutMs < 1)
|
|
247
850
|
throw new Error('queueTimeoutMs must be positive');
|
|
851
|
+
if (!this.policy.managed && request.allowance)
|
|
852
|
+
throw new Error('allowance requires managed admission');
|
|
853
|
+
if (!this.policy.managed && request.managed)
|
|
854
|
+
throw new Error('managed request requires an activated managed policy');
|
|
855
|
+
if (this.policy.managed) {
|
|
856
|
+
if ('projectId' in request)
|
|
857
|
+
throw new Error('request cannot choose its authority project');
|
|
858
|
+
if (request.allowance) {
|
|
859
|
+
if (request.allowance.runId !== request.orchestratorId || !request.allowance.taskId?.trim())
|
|
860
|
+
throw new Error('stable allowance run/task identity is required');
|
|
861
|
+
if (!Number.isSafeInteger(request.allowance.attempts ?? 1) || (request.allowance.attempts ?? 1) < 1
|
|
862
|
+
|| ((request.allowance.attempts ?? 1) !== 1 && request.managed?.kind !== 'native'))
|
|
863
|
+
throw new Error('attempt count requires a bounded native loop');
|
|
864
|
+
if (request.allowance.purpose !== undefined && !['initial', 'retry', 'fallback', 'quality', 'effort', 'review'].includes(request.allowance.purpose))
|
|
865
|
+
throw new Error('invalid attempt purpose');
|
|
866
|
+
validateBudgets(request.allowance.runBudgets);
|
|
867
|
+
validateBudgets(request.allowance.taskBudgets);
|
|
868
|
+
const reservation = request.allowance.reservation;
|
|
869
|
+
if (reservation !== undefined) {
|
|
870
|
+
if (!reservation || Object.keys(reservation).some(key => key !== 'tokens' && key !== 'costUsd'))
|
|
871
|
+
throw new Error('invalid allowance reservation');
|
|
872
|
+
for (const dimension of ['tokens', 'costUsd']) {
|
|
873
|
+
const value = reservation[dimension];
|
|
874
|
+
if (value !== null && (typeof value !== 'number' || !Number.isFinite(value) || value <= 0
|
|
875
|
+
|| (dimension === 'tokens' && !Number.isSafeInteger(value))))
|
|
876
|
+
throw new Error('invalid allowance reservation');
|
|
877
|
+
}
|
|
878
|
+
}
|
|
879
|
+
}
|
|
880
|
+
const worker = request.managed;
|
|
881
|
+
if (!worker || !['native', 'external'].includes(worker.kind))
|
|
882
|
+
throw new Error('managed worker identity is required');
|
|
883
|
+
if (!Number.isSafeInteger(worker.lowerBound ?? 1) || (worker.lowerBound ?? 1) < 1)
|
|
884
|
+
throw new Error('lowerBound must be a positive integer');
|
|
885
|
+
if (!Number.isFinite(request.queueTimeoutMs))
|
|
886
|
+
throw new Error('queueTimeoutMs must be finite');
|
|
887
|
+
managedDimensionKeys(this.policy.managed.projectId, worker);
|
|
888
|
+
if (worker.kind === 'external' && (!worker.group || !worker.member || !worker.cli || !worker.model || !worker.quotaScope)) {
|
|
889
|
+
throw new Error('external managed workers require group, member, installation, model and quota identities');
|
|
890
|
+
}
|
|
891
|
+
if (worker.cli) {
|
|
892
|
+
const binding = this.policy.managed.installations[worker.cli];
|
|
893
|
+
if (!binding || worker.quotaScope !== binding.quotaScope)
|
|
894
|
+
throw new Error('worker quota does not match the activated installation binding');
|
|
895
|
+
const upstream = this.policy.managed.upstream?.[worker.cli];
|
|
896
|
+
if (upstream && worker.model && !upstream.models.includes(worker.model))
|
|
897
|
+
throw new Error('worker model is not approved by the host installation binding');
|
|
898
|
+
}
|
|
899
|
+
else if (worker.quotaScope)
|
|
900
|
+
throw new Error('quota identity requires an installation binding');
|
|
901
|
+
if (worker.group || worker.member) {
|
|
902
|
+
const member = this.policy.managed.members[`${worker.group}/${worker.member}`];
|
|
903
|
+
if (!member || member.cli !== worker.cli)
|
|
904
|
+
throw new Error('worker does not match the activated group/member binding');
|
|
905
|
+
}
|
|
906
|
+
}
|
|
907
|
+
}
|
|
908
|
+
assertPolicyBinding(snapshot) {
|
|
909
|
+
if (snapshot.authorityRedirect)
|
|
910
|
+
throw new Error('project authority migrated; use the trusted host binding');
|
|
911
|
+
if (snapshot.hostBinding) {
|
|
912
|
+
if (snapshot.hostRegistrationRevoked)
|
|
913
|
+
throw new Error('trusted host executor registration is revoked');
|
|
914
|
+
const managed = this.policy.managed;
|
|
915
|
+
if (!managed || snapshot.hostBinding.hostId !== hostname()
|
|
916
|
+
|| canonical(hostProjectAdmissionPolicy(snapshot, managed.projectId)) !== canonical(this.policy)) {
|
|
917
|
+
throw new Error('trusted host policy revision or project binding mismatch');
|
|
918
|
+
}
|
|
919
|
+
}
|
|
920
|
+
else if (this.policy.managed || snapshot.policyBinding) {
|
|
921
|
+
if (!snapshot.policyBinding || canonical(snapshot.policyBinding) !== canonical(this.policy)) {
|
|
922
|
+
throw new Error('managed admission policy digest or policy binding mismatch');
|
|
923
|
+
}
|
|
924
|
+
}
|
|
925
|
+
}
|
|
926
|
+
assertSubmissionsOpen(snapshot) {
|
|
927
|
+
if (snapshot.migration || snapshot.authorityRedirect)
|
|
928
|
+
throw new Error('admission authority is frozen for migration');
|
|
929
|
+
}
|
|
930
|
+
requestKey(requestId) {
|
|
931
|
+
return this.policy.managed?.authorityRef.startsWith('host:')
|
|
932
|
+
? hostProjectKey(this.policy.managed.projectId, requestId) : requestId;
|
|
933
|
+
}
|
|
934
|
+
allowanceKey(key) {
|
|
935
|
+
return this.policy.managed?.authorityRef.startsWith('host:') ? hostProjectKey(this.policy.managed.projectId, key) : key;
|
|
936
|
+
}
|
|
937
|
+
policyFor(snapshot, record) {
|
|
938
|
+
return snapshot.hostBinding ? hostProjectAdmissionPolicy(snapshot, record.projectId) : this.policy;
|
|
939
|
+
}
|
|
940
|
+
reservationDimensions(request) {
|
|
941
|
+
const managed = this.policy.managed;
|
|
942
|
+
const worker = request.managed;
|
|
943
|
+
let dimensions = managedDimensionKeys(managed.projectId, worker);
|
|
944
|
+
if (managed.authorityRef.startsWith('host:'))
|
|
945
|
+
dimensions = hostReservationDimensions(managed, worker, dimensions);
|
|
946
|
+
// Native workers with incomplete identity conservatively occupy every unknown
|
|
947
|
+
// dimension. Missing observations must never manufacture extra capacity.
|
|
948
|
+
if (worker.kind === 'native') {
|
|
949
|
+
for (const key of Object.keys(managed.dimensionCaps)) {
|
|
950
|
+
if (managed.authorityRef.startsWith('host:') && key.startsWith('project/') && !key.startsWith(`project/${managed.projectId}/`))
|
|
951
|
+
continue;
|
|
952
|
+
if ((!worker.group && key.includes('/group/')) || (!worker.cli && key.includes('/installation/'))
|
|
953
|
+
|| (!worker.model && (key.startsWith('model/') || key.includes('/model/'))) || (!worker.quotaScope && (key.startsWith('quota/') || key.includes('/quota/'))))
|
|
954
|
+
dimensions.push(key);
|
|
955
|
+
}
|
|
956
|
+
}
|
|
957
|
+
return [...new Set(dimensions)].sort();
|
|
958
|
+
}
|
|
959
|
+
}
|
|
960
|
+
function hostProjectKey(projectId, key) {
|
|
961
|
+
return JSON.stringify([projectId, key]);
|
|
962
|
+
}
|
|
963
|
+
function isHostDimension(key) {
|
|
964
|
+
const match = /^project\/([a-f0-9]{64})(?:\/(model\/.*|quota\/.*))?$/.exec(key);
|
|
965
|
+
if (match)
|
|
966
|
+
return !match[2] || isCanonicalManagedDimension(match[1], match[2]);
|
|
967
|
+
const project = /^project\/([a-f0-9]{64})\//.exec(key)?.[1] ?? '0'.repeat(64);
|
|
968
|
+
return isCanonicalManagedDimension(project, key);
|
|
969
|
+
}
|
|
970
|
+
function hostReservationDimensions(policy, worker, dimensions) {
|
|
971
|
+
const project = `project/${policy.projectId}`;
|
|
972
|
+
const upstream = worker.cli ? policy.upstream[worker.cli] : undefined;
|
|
973
|
+
const keys = [...dimensions.filter(key => !key.startsWith('quota/')),
|
|
974
|
+
...(worker.model ? [`${project}/model/${worker.model}`] : []),
|
|
975
|
+
...(worker.quotaScope ? [`${project}/quota/${encodeURIComponent(worker.quotaScope)}`] : []),
|
|
976
|
+
...(upstream ? [`quota/${encodeURIComponent(upstream.quotaScope)}`] : []),
|
|
977
|
+
];
|
|
978
|
+
if (worker.kind === 'native')
|
|
979
|
+
for (const key of Object.keys(policy.dimensionCaps)) {
|
|
980
|
+
if (key.startsWith('project/') && !key.startsWith(`${project}/`))
|
|
981
|
+
continue;
|
|
982
|
+
if ((!worker.group && key.includes('/group/')) || (!worker.cli && key.includes('/installation/'))
|
|
983
|
+
|| (!worker.model && (key.startsWith('model/') || key.includes('/model/')))
|
|
984
|
+
|| (!worker.quotaScope && (key.startsWith('quota/') || key.includes('/quota/'))))
|
|
985
|
+
keys.push(key);
|
|
986
|
+
}
|
|
987
|
+
return [...new Set(keys)].sort();
|
|
988
|
+
}
|
|
989
|
+
/** Derive the exact per-project view of the persisted authority. No caller caps
|
|
990
|
+
* enter this function; project ceilings are approved by the host owner first. */
|
|
991
|
+
export function hostProjectAdmissionPolicy(snapshot, projectId) {
|
|
992
|
+
const host = snapshot.hostBinding;
|
|
993
|
+
const project = host?.projects[projectId];
|
|
994
|
+
const generation = snapshot.projectGenerations?.[projectId];
|
|
995
|
+
if (!host || !project || !generation)
|
|
996
|
+
throw new Error('missing approved host project binding or migration generation');
|
|
997
|
+
const dimensions = { ...host.dimensionCaps };
|
|
998
|
+
for (const [id, approved] of Object.entries(host.projects)) {
|
|
999
|
+
dimensions[`project/${id}`] = approved.policy.maxConcurrent;
|
|
1000
|
+
for (const [key, cap] of Object.entries(approved.policy.managed.dimensionCaps)) {
|
|
1001
|
+
const scoped = key.startsWith('model/') || key.startsWith('quota/') ? `project/${id}/${key}` : key;
|
|
1002
|
+
dimensions[scoped] = Math.min(dimensions[scoped] ?? cap, cap);
|
|
1003
|
+
}
|
|
1004
|
+
}
|
|
1005
|
+
const managed = project.policy.managed;
|
|
1006
|
+
const budgets = host.budgets || managed.budgets ? {
|
|
1007
|
+
...managed.budgets,
|
|
1008
|
+
enforcement: host.budgets?.enforcement === 'hard' || managed.budgets?.enforcement === 'hard' ? 'hard' : 'observed',
|
|
1009
|
+
unknownCost: host.budgets?.unknownCost === 'deny' || managed.budgets?.unknownCost === 'deny' ? 'deny' : 'reserve',
|
|
1010
|
+
reservationUsd: Math.max(host.budgets?.reservationUsd ?? 0, managed.budgets?.reservationUsd ?? 0) || undefined,
|
|
1011
|
+
...Object.fromEntries(['perRun', 'perTask'].map(scope => [scope, {
|
|
1012
|
+
...Object.fromEntries(['tokens', 'costUsd', 'wallSeconds'].flatMap(key => {
|
|
1013
|
+
const value = Math.min(host.budgets?.[scope]?.[key] ?? Infinity, managed.budgets?.[scope]?.[key] ?? Infinity);
|
|
1014
|
+
return Number.isFinite(value) ? [[key, value]] : [];
|
|
1015
|
+
})),
|
|
1016
|
+
}])),
|
|
1017
|
+
} : undefined;
|
|
1018
|
+
const retry = host.retry || managed.retry ? {
|
|
1019
|
+
maxAttempts: Math.min(host.retry?.maxAttempts ?? 3, managed.retry?.maxAttempts ?? 3),
|
|
1020
|
+
baseDelaySeconds: Math.max(host.retry?.baseDelaySeconds ?? 1, managed.retry?.baseDelaySeconds ?? 1),
|
|
1021
|
+
maxDelaySeconds: Math.max(host.retry?.maxDelaySeconds ?? 60, managed.retry?.maxDelaySeconds ?? 60),
|
|
1022
|
+
on: (managed.retry?.on ?? ['rate-limit', 'unavailable', 'timeout'])
|
|
1023
|
+
.filter(value => (host.retry?.on ?? ['rate-limit', 'unavailable', 'timeout']).includes(value)),
|
|
1024
|
+
} : undefined;
|
|
1025
|
+
return {
|
|
1026
|
+
...structuredClone(host.policy),
|
|
1027
|
+
leaseTtlMs: Math.min(host.policy.leaseTtlMs, project.policy.leaseTtlMs),
|
|
1028
|
+
managed: {
|
|
1029
|
+
...structuredClone(managed), authorityRef: host.authorityRef, hostPolicyRevision: host.revision,
|
|
1030
|
+
authorityGeneration: generation, maxQueued: Math.min(host.maxQueued, managed.maxQueued),
|
|
1031
|
+
dimensionCaps: dimensions, budgets, retry, upstream: structuredClone(project.installations),
|
|
1032
|
+
},
|
|
1033
|
+
};
|
|
1034
|
+
}
|
|
1035
|
+
/** Owner-only activation. Approved projects stay closed until their local store
|
|
1036
|
+
* (including an empty store) has completed freeze/drain/reconcile migration. */
|
|
1037
|
+
export function bindTrustedHostAdmissionPolicy(store, binding) {
|
|
1038
|
+
if (store instanceof FileAdmissionStore)
|
|
1039
|
+
store.useDurableWrites();
|
|
1040
|
+
const host = structuredClone(binding);
|
|
1041
|
+
if (!/^host:[a-z][a-z0-9-]{0,62}$/.test(host.authorityRef) || host.hostId !== hostname())
|
|
1042
|
+
throw new Error('host authority must be local to this execution host');
|
|
1043
|
+
if (!Number.isSafeInteger(host.revision) || host.revision < 1 || !Number.isSafeInteger(host.maxQueued) || host.maxQueued < 0)
|
|
1044
|
+
throw new Error('invalid host policy revision or queue bound');
|
|
1045
|
+
if ('managed' in host.policy)
|
|
1046
|
+
throw new Error('host policy cannot supply a caller managed binding');
|
|
1047
|
+
for (const [key, cap] of Object.entries(host.dimensionCaps)) {
|
|
1048
|
+
if ((!key.startsWith('model/') && !key.startsWith('quota/')) || !isHostDimension(key) || !Number.isSafeInteger(cap) || cap < 1)
|
|
1049
|
+
throw new Error('invalid host model/quota cap');
|
|
1050
|
+
}
|
|
1051
|
+
const accounts = new Map();
|
|
1052
|
+
for (const [id, approved] of Object.entries(host.projects)) {
|
|
1053
|
+
const policy = approved.policy;
|
|
1054
|
+
if (policy.managed?.authorityRef !== 'project' || policy.managed.projectId !== id)
|
|
1055
|
+
throw new Error('invalid approved project policy');
|
|
1056
|
+
new SharedHostScheduler(new InMemoryAdmissionStore(), policy);
|
|
1057
|
+
if (approved.migrationFrom) {
|
|
1058
|
+
new SharedHostScheduler(new InMemoryAdmissionStore(), approved.migrationFrom);
|
|
1059
|
+
const prior = structuredClone(approved.migrationFrom), current = structuredClone(policy);
|
|
1060
|
+
if (prior.managed?.authorityRef !== 'project' || prior.managed.projectId !== id)
|
|
1061
|
+
throw new Error('invalid prior project activation');
|
|
1062
|
+
prior.managed.policyDigest = current.managed.policyDigest;
|
|
1063
|
+
if (canonical(prior) !== canonical(current))
|
|
1064
|
+
throw new Error('authority migration cannot change unrelated project policy');
|
|
1065
|
+
}
|
|
1066
|
+
if (canonical(Object.keys(approved.installations).sort()) !== canonical(Object.keys(policy.managed.installations).sort()))
|
|
1067
|
+
throw new Error('host must approve every project installation');
|
|
1068
|
+
const localAccounts = new Map();
|
|
1069
|
+
for (const [cli, upstream] of Object.entries(approved.installations)) {
|
|
1070
|
+
if (!upstream.accountScope?.trim() || !upstream.quotaScope?.trim() || !Array.isArray(upstream.models)
|
|
1071
|
+
|| upstream.models.some(model => !isCanonicalWorkerModelKey(model)))
|
|
1072
|
+
throw new Error('invalid qualified upstream identity');
|
|
1073
|
+
const localAccount = policy.managed.installations[cli].accountScope;
|
|
1074
|
+
const identity = canonical([upstream.accountScope, upstream.quotaScope]);
|
|
1075
|
+
if (localAccounts.has(localAccount) && localAccounts.get(localAccount) !== identity)
|
|
1076
|
+
throw new Error('conflicting project account aliases in host binding');
|
|
1077
|
+
localAccounts.set(localAccount, identity);
|
|
1078
|
+
const previous = accounts.get(upstream.accountScope);
|
|
1079
|
+
if (previous && previous !== upstream.quotaScope)
|
|
1080
|
+
throw new Error('conflicting upstream account quota aliases');
|
|
1081
|
+
accounts.set(upstream.accountScope, upstream.quotaScope);
|
|
1082
|
+
}
|
|
1083
|
+
const view = hostProjectAdmissionPolicy({ revision: 0, records: {}, hostBinding: host, projectGenerations: { [id]: 1 } }, id);
|
|
1084
|
+
new SharedHostScheduler(new InMemoryAdmissionStore(), view);
|
|
1085
|
+
}
|
|
1086
|
+
store.transact(snapshot => {
|
|
1087
|
+
if (snapshot.migration || snapshot.authorityRedirect || snapshot.policyBinding || Object.keys(snapshot.records).length
|
|
1088
|
+
|| Object.keys(snapshot.allowances ?? {}).length || Object.keys(snapshot.cooldowns ?? {}).length)
|
|
1089
|
+
throw new Error('host authority activation requires an empty unbound store');
|
|
1090
|
+
if (snapshot.hostBinding && canonical(snapshot.hostBinding) !== canonical(host))
|
|
1091
|
+
throw new Error('host authority is already bound to another policy revision');
|
|
1092
|
+
snapshot.hostBinding = host;
|
|
1093
|
+
snapshot.projectGenerations ??= {};
|
|
1094
|
+
});
|
|
1095
|
+
}
|
|
1096
|
+
function migrationContentDigest(snapshot) {
|
|
1097
|
+
const { revision: _revision, migration: _migration, authorityRedirect: _redirect, ...content } = snapshot;
|
|
1098
|
+
return createHash('sha256').update(canonical(content)).digest('hex');
|
|
1099
|
+
}
|
|
1100
|
+
/** Trusted owner operation on the same AdmissionStore implementation. Interruptions
|
|
1101
|
+
* remain closed. Resume the same id/binding; no stale-lock deletion or blind rollback. */
|
|
1102
|
+
export function migrateProjectAdmissionToHost(source, target, projectId, options) {
|
|
1103
|
+
if (source instanceof FileAdmissionStore)
|
|
1104
|
+
source.useDurableWrites();
|
|
1105
|
+
if (target instanceof FileAdmissionStore)
|
|
1106
|
+
target.useDurableWrites();
|
|
1107
|
+
if (source === target || !options.id.trim())
|
|
1108
|
+
throw new Error('migration requires distinct authorities and a stable id');
|
|
1109
|
+
let local = source.read();
|
|
1110
|
+
let host = target.read();
|
|
1111
|
+
if (host.hostBinding?.hostId !== hostname())
|
|
1112
|
+
throw new Error('migration requires a local trusted host binding');
|
|
1113
|
+
const approved = host.hostBinding.projects[projectId];
|
|
1114
|
+
if (!approved || canonical(approved.migrationFrom ?? approved.policy) !== canonical(local.policyBinding))
|
|
1115
|
+
throw new Error('migration project policy is not host-approved');
|
|
1116
|
+
const bindingDigest = createHash('sha256').update(canonical(host.hostBinding)).digest('hex');
|
|
1117
|
+
const matches = (marker) => !!marker && marker.id === options.id
|
|
1118
|
+
&& marker.projectId === projectId && marker.bindingDigest === bindingDigest;
|
|
1119
|
+
if (local.authorityRedirect && !host.migration) {
|
|
1120
|
+
if (local.authorityRedirect.migrationId === options.id && local.authorityRedirect.authorityRef === host.hostBinding.authorityRef
|
|
1121
|
+
&& host.projectGenerations?.[projectId] === local.authorityRedirect.generation)
|
|
1122
|
+
return;
|
|
1123
|
+
throw new Error('project authority has already migrated');
|
|
1124
|
+
}
|
|
1125
|
+
if ((local.migration && !matches(local.migration)) || (host.migration && !matches(host.migration)))
|
|
1126
|
+
throw new Error('conflicting migration id or binding digest');
|
|
1127
|
+
if (!local.migration && !host.migration) {
|
|
1128
|
+
if (local.revision !== options.expectedSourceRevision || host.revision !== options.expectedTargetRevision)
|
|
1129
|
+
throw new Error('migration authority revision mismatch');
|
|
1130
|
+
if (host.projectGenerations?.[projectId])
|
|
1131
|
+
throw new Error('project already has an active host generation');
|
|
1132
|
+
}
|
|
1133
|
+
const write = (store, expected, mutate) => {
|
|
1134
|
+
store.transact(snapshot => {
|
|
1135
|
+
if (snapshot.revision !== expected.revision)
|
|
1136
|
+
throw new Error('migration authority revision changed; both authorities remain frozen');
|
|
1137
|
+
mutate(snapshot);
|
|
1138
|
+
});
|
|
1139
|
+
};
|
|
1140
|
+
const marker = { id: options.id, projectId, bindingDigest, phase: 'frozen' };
|
|
1141
|
+
// Close the old route first. The new project route has no generation yet, so
|
|
1142
|
+
// a crash between these writes cannot expose two usable authorities.
|
|
1143
|
+
if (!local.migration)
|
|
1144
|
+
write(source, local, snapshot => { snapshot.migration = marker; });
|
|
1145
|
+
if (!host.migration)
|
|
1146
|
+
write(target, host, snapshot => { snapshot.migration = marker; });
|
|
1147
|
+
local = source.read();
|
|
1148
|
+
host = target.read();
|
|
1149
|
+
let phase = host.migration.phase;
|
|
1150
|
+
if (phase === 'frozen') {
|
|
1151
|
+
options.onPhase?.('frozen');
|
|
1152
|
+
if ([...Object.values(local.records), ...Object.values(host.records)].some(record => record.state === 'admitted' || record.state === 'uncertain')) {
|
|
1153
|
+
throw new Error('migration requires proof-drained leases; live or uncertain work remains frozen');
|
|
1154
|
+
}
|
|
1155
|
+
const drained = { ...marker, phase: 'drained', sourceDigest: migrationContentDigest(local), targetDigest: migrationContentDigest(host) };
|
|
1156
|
+
write(source, local, snapshot => { snapshot.migration = drained; });
|
|
1157
|
+
write(target, host, snapshot => { snapshot.migration = drained; });
|
|
1158
|
+
local = source.read();
|
|
1159
|
+
host = target.read();
|
|
1160
|
+
phase = 'drained';
|
|
1161
|
+
}
|
|
1162
|
+
if (phase === 'drained') {
|
|
1163
|
+
options.onPhase?.('drained');
|
|
1164
|
+
if (migrationContentDigest(local) !== host.migration.sourceDigest || migrationContentDigest(host) !== host.migration.targetDigest)
|
|
1165
|
+
throw new Error('migration history digest changed; reconcile under the trusted owner');
|
|
1166
|
+
write(target, host, snapshot => {
|
|
1167
|
+
const view = hostProjectAdmissionPolicy({ ...snapshot, projectGenerations: { ...snapshot.projectGenerations, [projectId]: 1 } }, projectId);
|
|
1168
|
+
const scopes = snapshot.allowances ??= {};
|
|
1169
|
+
for (const [key, scope] of Object.entries(local.allowances ?? {})) {
|
|
1170
|
+
const scoped = hostProjectKey(projectId, key);
|
|
1171
|
+
if (scopes[scoped])
|
|
1172
|
+
throw new Error('migration allowance history conflicts');
|
|
1173
|
+
const budget = JSON.parse(key)[0] === 'run' ? view.managed.budgets?.perRun : view.managed.budgets?.perTask;
|
|
1174
|
+
const limits = intersectBudgets(scope.limits, projectBudgets(budget), JSON.parse(key)[0] === 'task' ? { maxAttempts: view.managed.retry?.maxAttempts ?? 3 } : undefined);
|
|
1175
|
+
const deadline = Math.min(Date.parse(scope.deadlineAt ?? '') || Infinity, limits.timeoutSeconds === undefined ? Infinity : Date.parse(scope.startedAt) + limits.timeoutSeconds * 1000);
|
|
1176
|
+
scopes[scoped] = { ...structuredClone(scope), limits, ...(Number.isFinite(deadline) ? { deadlineAt: new Date(deadline).toISOString() } : {}) };
|
|
1177
|
+
}
|
|
1178
|
+
for (const record of Object.values(local.records)) {
|
|
1179
|
+
if (!record.managed || !record.accounting || (record.accounting.attemptNumber !== undefined && (record.state !== 'settled' || !record.accounting.stopEvidence)))
|
|
1180
|
+
throw new Error('history requires proof-checked managed reconciliation');
|
|
1181
|
+
if (record.accounting.stopEvidence) {
|
|
1182
|
+
assertValidWorkerPayload('workerHealthSummary', record.accounting.stopEvidence);
|
|
1183
|
+
if (record.accounting.stopEvidence.observerGeneration !== record.leaseGeneration || decideLeaseMapping({ leaseGeneration: record.leaseGeneration,
|
|
1184
|
+
observedGeneration: record.leaseGeneration, leaseState: 'admitted', settled: false,
|
|
1185
|
+
health: record.accounting.stopEvidence }) !== 'release-once')
|
|
1186
|
+
throw new Error('migration requires current-generation confirmed stop evidence');
|
|
1187
|
+
}
|
|
1188
|
+
const key = hostProjectKey(projectId, record.requestId);
|
|
1189
|
+
if (snapshot.records[key])
|
|
1190
|
+
throw new Error('migration idempotency history conflicts');
|
|
1191
|
+
const imported = structuredClone(record);
|
|
1192
|
+
imported.projectId = projectId;
|
|
1193
|
+
if (imported.accounting) {
|
|
1194
|
+
imported.accounting.runKey = hostProjectKey(projectId, imported.accounting.runKey);
|
|
1195
|
+
imported.accounting.taskKey = hostProjectKey(projectId, imported.accounting.taskKey);
|
|
1196
|
+
}
|
|
1197
|
+
if (imported.managed)
|
|
1198
|
+
imported.dimensions = hostReservationDimensions(view.managed, imported.managed, managedDimensionKeys(projectId, imported.managed));
|
|
1199
|
+
snapshot.records[key] = imported;
|
|
1200
|
+
}
|
|
1201
|
+
const cooldowns = snapshot.cooldowns ??= {};
|
|
1202
|
+
for (const [key, cooldown] of Object.entries(local.cooldowns ?? {})) {
|
|
1203
|
+
if (!Number.isFinite(Date.parse(cooldown.until)) || !Number.isSafeInteger(cooldown.failures) || cooldown.failures < 0)
|
|
1204
|
+
throw new Error('invalid migration cooldown history');
|
|
1205
|
+
const aliases = Object.entries((approved.migrationFrom ?? approved.policy).managed.installations).filter(([, entry]) => `quota/${encodeURIComponent(entry.quotaScope)}` === key);
|
|
1206
|
+
if (!aliases.length)
|
|
1207
|
+
throw new Error('unbound cooldown history requires explicit reconciliation');
|
|
1208
|
+
const upstreamKeys = new Set(aliases.map(([cli]) => `quota/${encodeURIComponent(approved.installations[cli].quotaScope)}`));
|
|
1209
|
+
for (const upstreamKey of upstreamKeys) {
|
|
1210
|
+
const previous = cooldowns[upstreamKey];
|
|
1211
|
+
cooldowns[upstreamKey] = { until: previous && Date.parse(previous.until) > Date.parse(cooldown.until) ? previous.until : cooldown.until,
|
|
1212
|
+
failures: (previous?.failures ?? 0) + cooldown.failures };
|
|
1213
|
+
}
|
|
1214
|
+
}
|
|
1215
|
+
snapshot.migration = { ...snapshot.migration, phase: 'reconciled' };
|
|
1216
|
+
snapshot.migration.targetDigest = migrationContentDigest(snapshot);
|
|
1217
|
+
});
|
|
1218
|
+
host = target.read();
|
|
1219
|
+
phase = 'reconciled';
|
|
1220
|
+
}
|
|
1221
|
+
if (phase === 'reconciled') {
|
|
1222
|
+
options.onPhase?.('reconciled');
|
|
1223
|
+
local = source.read();
|
|
1224
|
+
if (migrationContentDigest(local) !== host.migration.sourceDigest || migrationContentDigest(host) !== host.migration.targetDigest)
|
|
1225
|
+
throw new Error('migration history digest changed before commit');
|
|
1226
|
+
write(target, host, snapshot => {
|
|
1227
|
+
(snapshot.projectGenerations ??= {})[projectId] = 1;
|
|
1228
|
+
snapshot.migration = { ...snapshot.migration, phase: 'committed' };
|
|
1229
|
+
snapshot.migration.targetDigest = migrationContentDigest(snapshot);
|
|
1230
|
+
});
|
|
1231
|
+
host = target.read();
|
|
1232
|
+
phase = 'committed';
|
|
248
1233
|
}
|
|
1234
|
+
if (phase === 'committed') {
|
|
1235
|
+
options.onPhase?.('committed');
|
|
1236
|
+
local = source.read();
|
|
1237
|
+
if (migrationContentDigest(local) !== host.migration.sourceDigest || migrationContentDigest(host) !== host.migration.targetDigest)
|
|
1238
|
+
throw new Error('history changed after migration commit');
|
|
1239
|
+
write(source, local, snapshot => {
|
|
1240
|
+
snapshot.authorityRedirect = { authorityRef: host.hostBinding.authorityRef, generation: host.projectGenerations[projectId], migrationId: options.id };
|
|
1241
|
+
});
|
|
1242
|
+
write(target, host, snapshot => { delete snapshot.migration; });
|
|
1243
|
+
}
|
|
1244
|
+
}
|
|
1245
|
+
function canonical(value) {
|
|
1246
|
+
const sorted = (node) => Array.isArray(node) ? node.map(sorted)
|
|
1247
|
+
: node !== null && typeof node === 'object'
|
|
1248
|
+
? Object.fromEntries(Object.keys(node).sort().filter(key => node[key] !== undefined).map(key => [key, sorted(node[key])])) : node;
|
|
1249
|
+
return JSON.stringify(sorted(value));
|
|
249
1250
|
}
|
|
250
|
-
function
|
|
1251
|
+
function isCanonicalManagedDimension(projectId, key) {
|
|
1252
|
+
const project = `project/${projectId}`;
|
|
1253
|
+
if (key === project)
|
|
1254
|
+
return true;
|
|
1255
|
+
if (key.startsWith(`${project}/`)) {
|
|
1256
|
+
return /^(group\/[a-z][a-z0-9-]{0,62}(\/member\/[a-z][a-z0-9-]{0,62})?|installation\/[a-z][a-z0-9-]{0,62})$/.test(key.slice(project.length + 1));
|
|
1257
|
+
}
|
|
1258
|
+
if (key.startsWith('model/'))
|
|
1259
|
+
return isCanonicalWorkerModelKey(key.slice(6));
|
|
1260
|
+
if (key.startsWith('quota/')) {
|
|
1261
|
+
try {
|
|
1262
|
+
return managedDimensionKeys(projectId, { kind: 'native', quotaScope: decodeURIComponent(key.slice(6)) }).at(-1) === key;
|
|
1263
|
+
}
|
|
1264
|
+
catch {
|
|
1265
|
+
return false;
|
|
1266
|
+
}
|
|
1267
|
+
}
|
|
1268
|
+
return false;
|
|
1269
|
+
}
|
|
1270
|
+
function sameManagedRequest(record, request) {
|
|
1271
|
+
const { state: _state, revision: _revision, reason: _reason, admittedAt: _admittedAt, leaseExpiresAt: _leaseExpiresAt, finishedAt: _finishedAt, preemptedBy: _preemptedBy, projectId: _project, attemptGeneration: _attempt, leaseGeneration: _lease, dimensions: _dimensions, reservedSlots: _slots, accounting: _accounting, launchClaimedAt: _launch, execution: _execution, health: _health, lastProgressAt: _progress, finalEvidence: _final, ...original } = record;
|
|
1272
|
+
return canonical(original) === canonical(request);
|
|
1273
|
+
}
|
|
1274
|
+
function belowQuota(active, field, value, quotas, slots = 1, managed = false) {
|
|
251
1275
|
const quota = quotas?.[String(value)];
|
|
252
1276
|
if (quota === undefined)
|
|
253
1277
|
return true;
|
|
254
|
-
|
|
1278
|
+
if (!managed)
|
|
1279
|
+
return active.filter(record => record[field] === value).length < quota;
|
|
1280
|
+
return active.filter(record => record[field] === value)
|
|
1281
|
+
.reduce((sum, record) => sum + (record.reservedSlots ?? 1), 0) + slots <= quota;
|
|
255
1282
|
}
|
|
256
1283
|
function sameRequest(record, request) {
|
|
257
1284
|
return record.orchestratorId === request.orchestratorId
|
|
@@ -259,4 +1286,48 @@ function sameRequest(record, request) {
|
|
|
259
1286
|
&& record.provider === request.provider
|
|
260
1287
|
&& record.runtimeKind === request.runtimeKind;
|
|
261
1288
|
}
|
|
1289
|
+
function validateBudgets(budgets) {
|
|
1290
|
+
for (const [key, value] of Object.entries(budgets ?? {})) {
|
|
1291
|
+
if (value === undefined)
|
|
1292
|
+
continue;
|
|
1293
|
+
if (!Number.isFinite(value) || value < 0 || (key === 'maxAttempts' && (!Number.isSafeInteger(value) || value < 1))) {
|
|
1294
|
+
throw new Error('invalid managed allowance ceiling');
|
|
1295
|
+
}
|
|
1296
|
+
}
|
|
1297
|
+
}
|
|
1298
|
+
function projectBudgets(budgets) {
|
|
1299
|
+
return { maxTokens: budgets?.tokens, maxCostUsd: budgets?.costUsd, timeoutSeconds: budgets?.wallSeconds };
|
|
1300
|
+
}
|
|
1301
|
+
function intersectBudgets(...budgets) {
|
|
1302
|
+
const result = {};
|
|
1303
|
+
for (const budget of budgets)
|
|
1304
|
+
for (const key of ['maxAttempts', 'maxTokens', 'maxCostUsd', 'timeoutSeconds']) {
|
|
1305
|
+
const value = budget?.[key];
|
|
1306
|
+
if (value !== undefined)
|
|
1307
|
+
result[key] = Math.min(result[key] ?? value, value);
|
|
1308
|
+
}
|
|
1309
|
+
return result;
|
|
1310
|
+
}
|
|
1311
|
+
function chargedUsage(record, dimension) {
|
|
1312
|
+
const accounting = record.accounting;
|
|
1313
|
+
const usage = accounting.usage;
|
|
1314
|
+
const measured = !usage || usage.quality === 'unknown' ? null : dimension === 'costUsd' ? usage.costUsd
|
|
1315
|
+
: usage.inputTokens === null || usage.outputTokens === null || usage.cacheTokens === null || usage.reasoningTokens === null ? null
|
|
1316
|
+
: usage.inputTokens + usage.outputTokens + usage.cacheTokens + usage.reasoningTokens;
|
|
1317
|
+
const reservation = accounting.reservation[dimension];
|
|
1318
|
+
// Uncertain execution can still accrue spend; estimates never refund a reserve.
|
|
1319
|
+
if (record.state !== 'settled' || usage?.quality !== 'reported') {
|
|
1320
|
+
return reservation === null || measured === null ? reservation : Math.max(reservation, measured);
|
|
1321
|
+
}
|
|
1322
|
+
return measured ?? reservation;
|
|
1323
|
+
}
|
|
1324
|
+
function canRefineUsage(previous, next) {
|
|
1325
|
+
const rank = { unknown: 0, estimated: 1, reported: 2 };
|
|
1326
|
+
if (rank[next.quality] < rank[previous.quality])
|
|
1327
|
+
return false;
|
|
1328
|
+
if (previous.quality !== 'reported')
|
|
1329
|
+
return true;
|
|
1330
|
+
return ['inputTokens', 'outputTokens', 'cacheTokens', 'reasoningTokens', 'costUsd']
|
|
1331
|
+
.every(key => previous[key] === null || (next[key] !== null && next[key] >= previous[key]));
|
|
1332
|
+
}
|
|
262
1333
|
//# sourceMappingURL=shared-host-scheduler.js.map
|