@osolmaz/pi-workflows 0.8.0 → 0.8.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -3
- package/dist/builtins/monitor.workflow.js +1 -0
- package/dist/builtins/monitor.workflow.js.map +1 -1
- package/docs/MONITOR.md +12 -9
- package/docs/WORKFLOW_UPDATES.md +8 -0
- package/docs/workflows.md +4 -12
- package/package.json +1 -5
- package/skills/monitor/SKILL.md +22 -15
- package/src/builtins/monitor.workflow.ts +1 -0
- package/dist/job-progress/index.d.ts +0 -3
- package/dist/job-progress/index.js +0 -4
- package/dist/job-progress/index.js.map +0 -1
- package/dist/job-progress/reporter.d.ts +0 -8
- package/dist/job-progress/reporter.js +0 -243
- package/dist/job-progress/reporter.js.map +0 -1
- package/dist/job-progress/types.d.ts +0 -65
- package/dist/job-progress/types.js +0 -2
- package/dist/job-progress/types.js.map +0 -1
- package/dist/job-progress/validation.d.ts +0 -5
- package/dist/job-progress/validation.js +0 -227
- package/dist/job-progress/validation.js.map +0 -1
- package/docs/JOB_PROGRESS.md +0 -119
- package/docs/plans/2026-08-17-durable-job-progress-plan.md +0 -176
- package/src/job-progress/index.ts +0 -24
- package/src/job-progress/reporter.ts +0 -304
- package/src/job-progress/types.ts +0 -92
- package/src/job-progress/validation.ts +0 -255
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
import type { ProgressEstimate, ProgressTrackState, WorkflowProgressData } from "../workflows/index.js";
|
|
2
|
-
export declare const JOB_PROGRESS_SCHEMA: "pi-workflows.job-progress.v1";
|
|
3
|
-
export type JobProgressState = "queued" | "running" | "waiting" | "blocked" | "completed" | "failed" | "cancelled" | "unknown";
|
|
4
|
-
export type JobProgressTrack = {
|
|
5
|
-
key: string;
|
|
6
|
-
data: WorkflowProgressData;
|
|
7
|
-
};
|
|
8
|
-
export type JobProgressCost = {
|
|
9
|
-
settledUsd: number;
|
|
10
|
-
reservedUsd: number;
|
|
11
|
-
};
|
|
12
|
-
export type JobProgressSnapshot = {
|
|
13
|
-
schema: typeof JOB_PROGRESS_SCHEMA;
|
|
14
|
-
application: string;
|
|
15
|
-
component: string;
|
|
16
|
-
jobId: string;
|
|
17
|
-
sourceRevision: string;
|
|
18
|
-
contractHash: string;
|
|
19
|
-
sequence: number;
|
|
20
|
-
state: JobProgressState;
|
|
21
|
-
phase: string;
|
|
22
|
-
startedAt: string;
|
|
23
|
-
updatedAt: string;
|
|
24
|
-
deadlineAt?: string;
|
|
25
|
-
finishedAt?: string;
|
|
26
|
-
tracks: JobProgressTrack[];
|
|
27
|
-
cost?: JobProgressCost;
|
|
28
|
-
};
|
|
29
|
-
export type JobProgressIdentity = Pick<JobProgressSnapshot, "application" | "component" | "jobId" | "sourceRevision" | "contractHash" | "startedAt"> & {
|
|
30
|
-
deadlineAt?: string;
|
|
31
|
-
};
|
|
32
|
-
export type JobProgressUpdate = {
|
|
33
|
-
state?: JobProgressState;
|
|
34
|
-
phase?: string;
|
|
35
|
-
tracks?: JobProgressTrack[];
|
|
36
|
-
cost?: JobProgressCost;
|
|
37
|
-
finishedAt?: string;
|
|
38
|
-
force?: boolean;
|
|
39
|
-
};
|
|
40
|
-
export type JobProgressPublish = (snapshot: JobProgressSnapshot, signal: AbortSignal) => Promise<void>;
|
|
41
|
-
type JobProgressReporterBaseOptions = JobProgressIdentity & {
|
|
42
|
-
initialState?: Exclude<JobProgressState, "completed" | "failed" | "cancelled">;
|
|
43
|
-
initialPhase?: string;
|
|
44
|
-
publish: JobProgressPublish;
|
|
45
|
-
minimumIntervalMs?: number;
|
|
46
|
-
publishTimeoutMs?: number;
|
|
47
|
-
now?: () => Date;
|
|
48
|
-
};
|
|
49
|
-
export type JobProgressReporterOptions = JobProgressReporterBaseOptions & ({
|
|
50
|
-
initialTracks: JobProgressTrack[];
|
|
51
|
-
previousSnapshot?: never;
|
|
52
|
-
} | {
|
|
53
|
-
initialTracks?: JobProgressTrack[];
|
|
54
|
-
previousSnapshot: JobProgressSnapshot;
|
|
55
|
-
});
|
|
56
|
-
export type JobProgressPublishResult = {
|
|
57
|
-
snapshot: JobProgressSnapshot;
|
|
58
|
-
published: boolean;
|
|
59
|
-
};
|
|
60
|
-
export type JobProgressEstimate = {
|
|
61
|
-
snapshot: JobProgressSnapshot;
|
|
62
|
-
tracks: ProgressTrackState[];
|
|
63
|
-
estimates: ProgressEstimate[];
|
|
64
|
-
};
|
|
65
|
-
export {};
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/job-progress/types.ts"],"names":[],"mappings":"AAMA,MAAM,CAAC,MAAM,mBAAmB,GAAG,8BAAuC,CAAC"}
|
|
@@ -1,5 +0,0 @@
|
|
|
1
|
-
import { type JobProgressSnapshot, type JobProgressState } from "./types.js";
|
|
2
|
-
export declare const MAX_JOB_PROGRESS_BYTES: number;
|
|
3
|
-
export declare const MAX_JOB_PROGRESS_TRACKS = 128;
|
|
4
|
-
export declare function validateJobProgressSnapshot(value: unknown): JobProgressSnapshot;
|
|
5
|
-
export declare function isTerminalJobProgressState(state: JobProgressState): state is "completed" | "failed" | "cancelled";
|
|
@@ -1,227 +0,0 @@
|
|
|
1
|
-
import { validateProgressData } from "../workflows/index.js";
|
|
2
|
-
import { JOB_PROGRESS_SCHEMA, } from "./types.js";
|
|
3
|
-
export const MAX_JOB_PROGRESS_BYTES = 64 * 1024;
|
|
4
|
-
export const MAX_JOB_PROGRESS_TRACKS = 128;
|
|
5
|
-
const SNAPSHOT_FIELDS = new Set([
|
|
6
|
-
"schema",
|
|
7
|
-
"application",
|
|
8
|
-
"component",
|
|
9
|
-
"jobId",
|
|
10
|
-
"sourceRevision",
|
|
11
|
-
"contractHash",
|
|
12
|
-
"sequence",
|
|
13
|
-
"state",
|
|
14
|
-
"phase",
|
|
15
|
-
"startedAt",
|
|
16
|
-
"updatedAt",
|
|
17
|
-
"deadlineAt",
|
|
18
|
-
"finishedAt",
|
|
19
|
-
"tracks",
|
|
20
|
-
"cost",
|
|
21
|
-
]);
|
|
22
|
-
const TRACK_FIELDS = new Set(["key", "data"]);
|
|
23
|
-
const COST_FIELDS = new Set(["settledUsd", "reservedUsd"]);
|
|
24
|
-
const STATES = new Set([
|
|
25
|
-
"queued",
|
|
26
|
-
"running",
|
|
27
|
-
"waiting",
|
|
28
|
-
"blocked",
|
|
29
|
-
"completed",
|
|
30
|
-
"failed",
|
|
31
|
-
"cancelled",
|
|
32
|
-
"unknown",
|
|
33
|
-
]);
|
|
34
|
-
const KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/;
|
|
35
|
-
const RFC_3339_PATTERN = /^(\d{4})-(\d{2})-(\d{2})T(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$/;
|
|
36
|
-
export function validateJobProgressSnapshot(value) {
|
|
37
|
-
const bytes = jsonBytes(value);
|
|
38
|
-
if (bytes > MAX_JOB_PROGRESS_BYTES) {
|
|
39
|
-
throw new Error(`job progress snapshot must be at most ${MAX_JOB_PROGRESS_BYTES} bytes`);
|
|
40
|
-
}
|
|
41
|
-
if (!isRecord(value))
|
|
42
|
-
throw new Error("job progress snapshot must be an object");
|
|
43
|
-
rejectUnknown(value, SNAPSHOT_FIELDS, "job progress");
|
|
44
|
-
if (value.schema !== JOB_PROGRESS_SCHEMA) {
|
|
45
|
-
throw new Error(`job progress.schema must equal ${JOB_PROGRESS_SCHEMA}`);
|
|
46
|
-
}
|
|
47
|
-
const application = requiredText(value.application, "job progress.application", 100);
|
|
48
|
-
const component = requiredText(value.component, "job progress.component", 100);
|
|
49
|
-
const jobId = requiredText(value.jobId, "job progress.jobId", 200);
|
|
50
|
-
const sourceRevision = requiredText(value.sourceRevision, "job progress.sourceRevision", 200);
|
|
51
|
-
const contractHash = requiredText(value.contractHash, "job progress.contractHash", 200);
|
|
52
|
-
const sequence = requiredInteger(value.sequence, "job progress.sequence");
|
|
53
|
-
const state = requiredState(value.state);
|
|
54
|
-
const phase = requiredText(value.phase, "job progress.phase", 100);
|
|
55
|
-
const startedAt = requiredTimestamp(value.startedAt, "job progress.startedAt");
|
|
56
|
-
const updatedAt = requiredTimestamp(value.updatedAt, "job progress.updatedAt");
|
|
57
|
-
const deadlineAt = optionalTimestamp(value.deadlineAt, "job progress.deadlineAt");
|
|
58
|
-
const finishedAt = optionalTimestamp(value.finishedAt, "job progress.finishedAt");
|
|
59
|
-
const tracks = requiredTracks(value.tracks);
|
|
60
|
-
const cost = optionalCost(value.cost);
|
|
61
|
-
if (Date.parse(updatedAt) < Date.parse(startedAt)) {
|
|
62
|
-
throw new Error("job progress.updatedAt must not be before startedAt");
|
|
63
|
-
}
|
|
64
|
-
if (deadlineAt !== undefined && Date.parse(deadlineAt) <= Date.parse(startedAt)) {
|
|
65
|
-
throw new Error("job progress.deadlineAt must be after startedAt");
|
|
66
|
-
}
|
|
67
|
-
if (finishedAt !== undefined && Date.parse(finishedAt) < Date.parse(startedAt)) {
|
|
68
|
-
throw new Error("job progress.finishedAt must not be before startedAt");
|
|
69
|
-
}
|
|
70
|
-
if (finishedAt !== undefined && Date.parse(finishedAt) > Date.parse(updatedAt)) {
|
|
71
|
-
throw new Error("job progress.finishedAt must not be after updatedAt");
|
|
72
|
-
}
|
|
73
|
-
if (isTerminal(state) && finishedAt === undefined) {
|
|
74
|
-
throw new Error("job progress.finishedAt is required for a terminal state");
|
|
75
|
-
}
|
|
76
|
-
if (!isTerminal(state) && finishedAt !== undefined) {
|
|
77
|
-
throw new Error("job progress.finishedAt is allowed only for a terminal state");
|
|
78
|
-
}
|
|
79
|
-
if (isTerminal(state) && tracks.some((track) => !isTerminalTrack(track))) {
|
|
80
|
-
throw new Error("job progress tracks must be terminal when the job state is terminal");
|
|
81
|
-
}
|
|
82
|
-
return {
|
|
83
|
-
schema: JOB_PROGRESS_SCHEMA,
|
|
84
|
-
application,
|
|
85
|
-
component,
|
|
86
|
-
jobId,
|
|
87
|
-
sourceRevision,
|
|
88
|
-
contractHash,
|
|
89
|
-
sequence,
|
|
90
|
-
state,
|
|
91
|
-
phase,
|
|
92
|
-
startedAt,
|
|
93
|
-
updatedAt,
|
|
94
|
-
...(deadlineAt === undefined ? {} : { deadlineAt }),
|
|
95
|
-
...(finishedAt === undefined ? {} : { finishedAt }),
|
|
96
|
-
tracks,
|
|
97
|
-
...(cost === undefined ? {} : { cost }),
|
|
98
|
-
};
|
|
99
|
-
}
|
|
100
|
-
export function isTerminalJobProgressState(state) {
|
|
101
|
-
return isTerminal(state);
|
|
102
|
-
}
|
|
103
|
-
function requiredTracks(value) {
|
|
104
|
-
if (!Array.isArray(value))
|
|
105
|
-
throw new Error("job progress.tracks must be an array");
|
|
106
|
-
if (value.length < 1)
|
|
107
|
-
throw new Error("job progress.tracks must contain at least one entry");
|
|
108
|
-
if (value.length > MAX_JOB_PROGRESS_TRACKS) {
|
|
109
|
-
throw new Error(`job progress.tracks must contain at most ${MAX_JOB_PROGRESS_TRACKS} entries`);
|
|
110
|
-
}
|
|
111
|
-
const keys = new Set();
|
|
112
|
-
return value.map((item, index) => {
|
|
113
|
-
if (!isRecord(item))
|
|
114
|
-
throw new Error(`job progress.tracks[${index}] must be an object`);
|
|
115
|
-
rejectUnknown(item, TRACK_FIELDS, `job progress.tracks[${index}]`);
|
|
116
|
-
if (typeof item.key !== "string" || !KEY_PATTERN.test(item.key)) {
|
|
117
|
-
throw new Error(`job progress.tracks[${index}].key must match [A-Za-z0-9][A-Za-z0-9._:/-]{0,127}`);
|
|
118
|
-
}
|
|
119
|
-
if (keys.has(item.key))
|
|
120
|
-
throw new Error(`job progress track key ${item.key} is duplicated`);
|
|
121
|
-
keys.add(item.key);
|
|
122
|
-
if (!isRecord(item.data)) {
|
|
123
|
-
throw new Error(`job progress.tracks[${index}].data must be an object`);
|
|
124
|
-
}
|
|
125
|
-
return { key: item.key, data: structuredClone(validateProgressData(item.data)) };
|
|
126
|
-
});
|
|
127
|
-
}
|
|
128
|
-
function optionalCost(value) {
|
|
129
|
-
if (value === undefined)
|
|
130
|
-
return undefined;
|
|
131
|
-
if (!isRecord(value))
|
|
132
|
-
throw new Error("job progress.cost must be an object");
|
|
133
|
-
rejectUnknown(value, COST_FIELDS, "job progress.cost");
|
|
134
|
-
return {
|
|
135
|
-
settledUsd: requiredNonNegative(value.settledUsd, "job progress.cost.settledUsd"),
|
|
136
|
-
reservedUsd: requiredNonNegative(value.reservedUsd, "job progress.cost.reservedUsd"),
|
|
137
|
-
};
|
|
138
|
-
}
|
|
139
|
-
function requiredState(value) {
|
|
140
|
-
if (typeof value !== "string" || !STATES.has(value)) {
|
|
141
|
-
throw new Error("job progress.state is invalid");
|
|
142
|
-
}
|
|
143
|
-
return value;
|
|
144
|
-
}
|
|
145
|
-
function requiredText(value, field, max) {
|
|
146
|
-
if (typeof value !== "string" || value.trim().length < 1 || value.trim().length > max) {
|
|
147
|
-
throw new Error(`${field} must be 1 to ${max} characters`);
|
|
148
|
-
}
|
|
149
|
-
if ([...value].some(isControlCharacter))
|
|
150
|
-
throw new Error(`${field} must not contain control characters`);
|
|
151
|
-
return value;
|
|
152
|
-
}
|
|
153
|
-
function requiredInteger(value, field) {
|
|
154
|
-
if (!Number.isSafeInteger(value) || value < 0) {
|
|
155
|
-
throw new Error(`${field} must be a non-negative safe integer`);
|
|
156
|
-
}
|
|
157
|
-
return value;
|
|
158
|
-
}
|
|
159
|
-
function requiredNonNegative(value, field) {
|
|
160
|
-
if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
|
|
161
|
-
throw new Error(`${field} must be a finite non-negative number`);
|
|
162
|
-
}
|
|
163
|
-
return value;
|
|
164
|
-
}
|
|
165
|
-
function requiredTimestamp(value, field) {
|
|
166
|
-
const parsed = optionalTimestamp(value, field);
|
|
167
|
-
if (parsed === undefined)
|
|
168
|
-
throw new Error(`${field} is required`);
|
|
169
|
-
return parsed;
|
|
170
|
-
}
|
|
171
|
-
function optionalTimestamp(value, field) {
|
|
172
|
-
if (value === undefined)
|
|
173
|
-
return undefined;
|
|
174
|
-
if (typeof value !== "string") {
|
|
175
|
-
throw new Error(`${field} must be an RFC 3339 timestamp with an offset`);
|
|
176
|
-
}
|
|
177
|
-
const match = RFC_3339_PATTERN.exec(value);
|
|
178
|
-
if (match === null || !validCalendarDate(match[1], match[2], match[3])) {
|
|
179
|
-
throw new Error(`${field} must be an RFC 3339 timestamp with an offset`);
|
|
180
|
-
}
|
|
181
|
-
return value;
|
|
182
|
-
}
|
|
183
|
-
function validCalendarDate(yearText, monthText, dayText) {
|
|
184
|
-
const year = Number(yearText);
|
|
185
|
-
const month = Number(monthText);
|
|
186
|
-
const day = Number(dayText);
|
|
187
|
-
if (!Number.isInteger(year) || year < 1 || month < 1 || month > 12 || day < 1)
|
|
188
|
-
return false;
|
|
189
|
-
return day <= new Date(Date.UTC(year, month, 0)).getUTCDate();
|
|
190
|
-
}
|
|
191
|
-
function rejectUnknown(value, allowed, field) {
|
|
192
|
-
for (const key of Object.keys(value)) {
|
|
193
|
-
if (!allowed.has(key))
|
|
194
|
-
throw new Error(`${field}.${key} is not supported`);
|
|
195
|
-
}
|
|
196
|
-
}
|
|
197
|
-
function isRecord(value) {
|
|
198
|
-
if (value === null || typeof value !== "object" || Array.isArray(value))
|
|
199
|
-
return false;
|
|
200
|
-
const prototype = Object.getPrototypeOf(value);
|
|
201
|
-
return prototype === Object.prototype || prototype === null;
|
|
202
|
-
}
|
|
203
|
-
function jsonBytes(value) {
|
|
204
|
-
let encoded;
|
|
205
|
-
try {
|
|
206
|
-
encoded = JSON.stringify(value);
|
|
207
|
-
}
|
|
208
|
-
catch {
|
|
209
|
-
throw new Error("job progress snapshot must be JSON serializable");
|
|
210
|
-
}
|
|
211
|
-
if (encoded === undefined)
|
|
212
|
-
throw new Error("job progress snapshot must be JSON serializable");
|
|
213
|
-
return Buffer.byteLength(encoded, "utf8");
|
|
214
|
-
}
|
|
215
|
-
function isControlCharacter(character) {
|
|
216
|
-
const code = character.codePointAt(0) ?? 0;
|
|
217
|
-
return code < 32 || (code >= 127 && code <= 159);
|
|
218
|
-
}
|
|
219
|
-
function isTerminal(state) {
|
|
220
|
-
return state === "completed" || state === "failed" || state === "cancelled";
|
|
221
|
-
}
|
|
222
|
-
function isTerminalTrack(track) {
|
|
223
|
-
return (track.data.status === "completed" ||
|
|
224
|
-
track.data.status === "failed" ||
|
|
225
|
-
track.data.status === "cancelled");
|
|
226
|
-
}
|
|
227
|
-
//# sourceMappingURL=validation.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"validation.js","sourceRoot":"","sources":["../../src/job-progress/validation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EACL,mBAAmB,GAKpB,MAAM,YAAY,CAAC;AAEpB,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,GAAG,IAAI,CAAC;AAChD,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAE3C,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,QAAQ;IACR,aAAa;IACb,WAAW;IACX,OAAO;IACP,gBAAgB;IAChB,cAAc;IACd,UAAU;IACV,OAAO;IACP,OAAO;IACP,WAAW;IACX,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,QAAQ;IACR,MAAM;CACP,CAAC,CAAC;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAC9C,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC,CAAC;AAC3D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAmB;IACvC,QAAQ;IACR,SAAS;IACT,SAAS;IACT,SAAS;IACT,WAAW;IACX,QAAQ;IACR,WAAW;IACX,SAAS;CACV,CAAC,CAAC;AACH,MAAM,WAAW,GAAG,sCAAsC,CAAC;AAC3D,MAAM,gBAAgB,GACpB,0GAA0G,CAAC;AAE7G,MAAM,UAAU,2BAA2B,CAAC,KAAc;IACxD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,KAAK,GAAG,sBAAsB,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,yCAAyC,sBAAsB,QAAQ,CAAC,CAAC;IAC3F,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IACjF,aAAa,CAAC,KAAK,EAAE,eAAe,EAAE,cAAc,CAAC,CAAC;IACtD,IAAI,KAAK,CAAC,MAAM,KAAK,mBAAmB,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,kCAAkC,mBAAmB,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,MAAM,WAAW,GAAG,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,0BAA0B,EAAE,GAAG,CAAC,CAAC;IACrF,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,EAAE,wBAAwB,EAAE,GAAG,CAAC,CAAC;IAC/E,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,oBAAoB,EAAE,GAAG,CAAC,CAAC;IACnE,MAAM,cAAc,GAAG,YAAY,CAAC,KAAK,CAAC,cAAc,EAAE,6BAA6B,EAAE,GAAG,CAAC,CAAC;IAC9F,MAAM,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,EAAE,2BAA2B,EAAE,GAAG,CAAC,CAAC;IACxF,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,uBAAuB,CAAC,CAAC;IAC1E,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,oBAAoB,EAAE,GAAG,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;IAC/E,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;IAC/E,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAC;IAClF,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAC;IAClF,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEtC,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAChF,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/E,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/E,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IAC9E,CAAC;IACD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;IAED,OAAO;QACL,MAAM,EAAE,mBAAmB;QAC3B,WAAW;QACX,SAAS;QACT,KAAK;QACL,cAAc;QACd,YAAY;QACZ,QAAQ;QACR,KAAK;QACL,KAAK;QACL,SAAS;QACT,SAAS;QACT,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;QACnD,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;QACnD,MAAM;QACN,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;KACxC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAuB;IAEvB,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACnF,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IAC7F,IAAI,KAAK,CAAC,MAAM,GAAG,uBAAuB,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,4CAA4C,uBAAuB,UAAU,CAAC,CAAC;IACjG,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,KAAK,qBAAqB,CAAC,CAAC;QACxF,aAAa,CAAC,IAAI,EAAE,YAAY,EAAE,uBAAuB,KAAK,GAAG,CAAC,CAAC;QACnE,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,QAAQ,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CACb,uBAAuB,KAAK,qDAAqD,CAClF,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,GAAG,gBAAgB,CAAC,CAAC;QAC5F,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,uBAAuB,KAAK,0BAA0B,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,eAAe,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;IACnF,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC7E,aAAa,CAAC,KAAK,EAAE,WAAW,EAAE,mBAAmB,CAAC,CAAC;IACvD,OAAO;QACL,UAAU,EAAE,mBAAmB,CAAC,KAAK,CAAC,UAAU,EAAE,8BAA8B,CAAC;QACjF,WAAW,EAAE,mBAAmB,CAAC,KAAK,CAAC,WAAW,EAAE,+BAA+B,CAAC;KACrF,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAyB,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,KAAyB,CAAC;AACnC,CAAC;AAED,SAAS,YAAY,CAAC,KAAc,EAAE,KAAa,EAAE,GAAW;IAC9D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACtF,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,iBAAiB,GAAG,aAAa,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,sCAAsC,CAAC,CAAC;IAClE,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,KAAa;IACpD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAK,KAAgB,GAAG,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,sCAAsC,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,KAAe,CAAC;AACzB,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc,EAAE,KAAa;IACxD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,uCAAuC,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAE,KAAa;IACtD,MAAM,MAAM,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC/C,IAAI,MAAM,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,cAAc,CAAC,CAAC;IAClE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAE,KAAa;IACtD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,+CAA+C,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3C,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,+CAA+C,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CACxB,QAA4B,EAC5B,SAA6B,EAC7B,OAA2B;IAE3B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAChC,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,EAAE,IAAI,GAAG,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5F,OAAO,GAAG,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC;AAChE,CAAC;AAED,SAAS,aAAa,CACpB,KAA8B,EAC9B,OAA4B,EAC5B,KAAa;IAEb,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,IAAI,GAAG,mBAAmB,CAAC,CAAC;IAC7E,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtF,MAAM,SAAS,GAAG,MAAM,CAAC,cAAc,CAAC,KAAK,CAAY,CAAC;IAC1D,OAAO,SAAS,KAAK,MAAM,CAAC,SAAS,IAAI,SAAS,KAAK,IAAI,CAAC;AAC9D,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAA2B,CAAC;IAChC,IAAI,CAAC;QACH,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,OAAO,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IAC9F,OAAO,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,kBAAkB,CAAC,SAAiB;IAC3C,MAAM,IAAI,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC3C,OAAO,IAAI,GAAG,EAAE,IAAI,CAAC,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,UAAU,CAAC,KAAuB;IACzC,OAAO,KAAK,KAAK,WAAW,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,WAAW,CAAC;AAC9E,CAAC;AAED,SAAS,eAAe,CAAC,KAAuB;IAC9C,OAAO,CACL,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW;QACjC,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ;QAC9B,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW,CAClC,CAAC;AACJ,CAAC"}
|
package/docs/JOB_PROGRESS.md
DELETED
|
@@ -1,119 +0,0 @@
|
|
|
1
|
-
# Durable job progress
|
|
2
|
-
|
|
3
|
-
`@osolmaz/pi-workflows/job-progress` lets a remote job publish progress that a Pi Workflows monitor can validate and measure. It uses the existing `pi-workflows.progress.v1` track contract and ETA estimator.
|
|
4
|
-
|
|
5
|
-
## Report progress
|
|
6
|
-
|
|
7
|
-
Create one reporter for one physical job. Inject the storage write so the package remains independent of a cloud provider.
|
|
8
|
-
|
|
9
|
-
```ts
|
|
10
|
-
import { createJobProgressReporter } from "@osolmaz/pi-workflows/job-progress";
|
|
11
|
-
|
|
12
|
-
const reporter = createJobProgressReporter({
|
|
13
|
-
application: "example",
|
|
14
|
-
component: "batch-worker",
|
|
15
|
-
jobId: process.env.JOB_ID ?? "local",
|
|
16
|
-
sourceRevision: "abc123",
|
|
17
|
-
contractHash: "def456",
|
|
18
|
-
startedAt: new Date().toISOString(),
|
|
19
|
-
initialTracks: [
|
|
20
|
-
{
|
|
21
|
-
key: "overall",
|
|
22
|
-
data: {
|
|
23
|
-
schema: "pi-workflows.progress.v1",
|
|
24
|
-
status: "running",
|
|
25
|
-
phase: "starting",
|
|
26
|
-
},
|
|
27
|
-
},
|
|
28
|
-
],
|
|
29
|
-
minimumIntervalMs: 30_000,
|
|
30
|
-
publishTimeoutMs: 15_000,
|
|
31
|
-
publish: async (snapshot, signal) => {
|
|
32
|
-
await bucket.writeText(progressPath, JSON.stringify(snapshot), { signal });
|
|
33
|
-
},
|
|
34
|
-
});
|
|
35
|
-
|
|
36
|
-
await reporter.report({
|
|
37
|
-
phase: "processing",
|
|
38
|
-
tracks: [
|
|
39
|
-
{
|
|
40
|
-
key: "records",
|
|
41
|
-
data: {
|
|
42
|
-
schema: "pi-workflows.progress.v1",
|
|
43
|
-
status: "running",
|
|
44
|
-
phase: "processing",
|
|
45
|
-
completed: 400,
|
|
46
|
-
total: 1_000,
|
|
47
|
-
unit: "records",
|
|
48
|
-
},
|
|
49
|
-
},
|
|
50
|
-
],
|
|
51
|
-
});
|
|
52
|
-
```
|
|
53
|
-
|
|
54
|
-
The first update, each phase change, and each terminal update publishes immediately. Other updates are coalesced until `minimumIntervalMs` has passed. Call `flush()` at a durable checkpoint or before exit when the current snapshot has not been published.
|
|
55
|
-
|
|
56
|
-
On process restart, read and validate the existing snapshot and pass it as `previousSnapshot`. The reporter continues its sequence number and rejects an identity, deadline, or terminal-state mismatch.
|
|
57
|
-
|
|
58
|
-
The reporter keeps the latest snapshot after a write failure. The application decides when to log and retry that failure. A timed-out storage write remains serialized until the underlying write settles, so an older write cannot overwrite a newer snapshot. Storage adapters must honor the abort signal and settle after cancellation. A progress failure must not replace receipt or checkpoint validation.
|
|
59
|
-
|
|
60
|
-
## Finish a job
|
|
61
|
-
|
|
62
|
-
A terminal update gets a finish timestamp and publishes immediately:
|
|
63
|
-
|
|
64
|
-
```ts
|
|
65
|
-
await reporter.report({
|
|
66
|
-
state: "completed",
|
|
67
|
-
phase: "complete",
|
|
68
|
-
tracks: [
|
|
69
|
-
{
|
|
70
|
-
key: "records",
|
|
71
|
-
data: {
|
|
72
|
-
schema: "pi-workflows.progress.v1",
|
|
73
|
-
status: "completed",
|
|
74
|
-
phase: "complete",
|
|
75
|
-
completed: 1_000,
|
|
76
|
-
total: 1_000,
|
|
77
|
-
unit: "records",
|
|
78
|
-
},
|
|
79
|
-
},
|
|
80
|
-
],
|
|
81
|
-
});
|
|
82
|
-
```
|
|
83
|
-
|
|
84
|
-
A terminal snapshot is not a receipt. Receipts, manifests, hashes, and durable application outputs remain authoritative.
|
|
85
|
-
|
|
86
|
-
## Store and discover snapshots
|
|
87
|
-
|
|
88
|
-
Write one mutable snapshot per physical job:
|
|
89
|
-
|
|
90
|
-
```text
|
|
91
|
-
<existing-bucket>/<application-prefix>/runs/<job-id>/progress.json
|
|
92
|
-
```
|
|
93
|
-
|
|
94
|
-
Add these immutable labels to the job or schedule:
|
|
95
|
-
|
|
96
|
-
```text
|
|
97
|
-
progress_schema=pi-workflows.job-progress.v1
|
|
98
|
-
progress_bucket=<existing-bucket>
|
|
99
|
-
progress_prefix=<application-prefix>/runs
|
|
100
|
-
```
|
|
101
|
-
|
|
102
|
-
A monitor reads the labels, forms `<progress_prefix>/<job-id>/progress.json`, validates the snapshot with `validateJobProgressSnapshot`, and publishes its tracks with stable keys. It should retain consecutive snapshots so Pi Workflows can estimate a conservative ETA from measured rates.
|
|
103
|
-
|
|
104
|
-
## Estimate from snapshots
|
|
105
|
-
|
|
106
|
-
```ts
|
|
107
|
-
import { estimateJobProgress } from "@osolmaz/pi-workflows/job-progress";
|
|
108
|
-
|
|
109
|
-
const result = estimateJobProgress(previousSnapshots);
|
|
110
|
-
for (const estimate of result.estimates) {
|
|
111
|
-
console.log(estimate.key, estimate.remainingMedianMs);
|
|
112
|
-
}
|
|
113
|
-
```
|
|
114
|
-
|
|
115
|
-
The estimator returns no measured ETA until a track has a known total and enough positive progress samples. A source-provided `sourceEstimatedFinishAt` remains available through the normal progress contract.
|
|
116
|
-
|
|
117
|
-
## Data boundary
|
|
118
|
-
|
|
119
|
-
Snapshots may contain identifiers, phases, counters, totals, timestamps, and cost totals. Do not put credentials, environment values, input records, model responses, logs, or private content in a snapshot. The strict validator rejects unknown fields and limits the encoded snapshot to 64 KiB.
|
|
@@ -1,176 +0,0 @@
|
|
|
1
|
-
# Durable job progress
|
|
2
|
-
|
|
3
|
-
## Problem
|
|
4
|
-
|
|
5
|
-
Long-running remote jobs can expose state only through logs and final receipts. A monitor can confirm that a job is running, but it cannot give a reliable progress value or remaining time when the job does not publish a completed count, a total, and source timestamps.
|
|
6
|
-
|
|
7
|
-
xTap Pool and OurModels need the same durable progress contract. The contract must work with their existing Hugging Face Buckets, survive worker restarts, and use the ETA estimator that Pi Workflows already uses for `pi-workflows.progress.v1` tracks.
|
|
8
|
-
|
|
9
|
-
## Requirements
|
|
10
|
-
|
|
11
|
-
The implementation must:
|
|
12
|
-
|
|
13
|
-
- add one storage-neutral job progress API to `@osolmaz/pi-workflows`
|
|
14
|
-
- reuse `pi-workflows.progress.v1` for progress tracks
|
|
15
|
-
- write one mutable snapshot for each physical job in the application's existing Bucket
|
|
16
|
-
- let monitors discover the snapshot from immutable job labels
|
|
17
|
-
- let Pi Workflows validate the snapshot and estimate remaining time from repeated samples
|
|
18
|
-
- let xTap Pool report restore, review, recovery, and publication progress
|
|
19
|
-
- let OurModels report discovery, processing, cache, and publication progress
|
|
20
|
-
- keep receipts, content hashes, manifests, and databases authoritative
|
|
21
|
-
- avoid secrets, post text, model output, and other private content in progress snapshots
|
|
22
|
-
- preserve one physical enrichment job at a time during the xTap Pool change
|
|
23
|
-
- leave the OurModels replacement schedule suspended until a paid run is separately approved
|
|
24
|
-
|
|
25
|
-
## Non-goals
|
|
26
|
-
|
|
27
|
-
This work does not add:
|
|
28
|
-
|
|
29
|
-
- a new remote store
|
|
30
|
-
- a metrics database
|
|
31
|
-
- a second ETA protocol
|
|
32
|
-
- a Pi core change
|
|
33
|
-
- a service or daemon
|
|
34
|
-
- a compatibility path for an older snapshot schema
|
|
35
|
-
- automatic authority to retry, deploy, publish, or spend money
|
|
36
|
-
|
|
37
|
-
## Public contract
|
|
38
|
-
|
|
39
|
-
The npm package exports a new subpath:
|
|
40
|
-
|
|
41
|
-
```ts
|
|
42
|
-
import {
|
|
43
|
-
createJobProgressReporter,
|
|
44
|
-
estimateJobProgress,
|
|
45
|
-
validateJobProgressSnapshot,
|
|
46
|
-
type JobProgressSnapshot,
|
|
47
|
-
} from "@osolmaz/pi-workflows/job-progress";
|
|
48
|
-
```
|
|
49
|
-
|
|
50
|
-
A snapshot has schema `pi-workflows.job-progress.v1` and contains:
|
|
51
|
-
|
|
52
|
-
- stable application and component names
|
|
53
|
-
- the physical job identifier
|
|
54
|
-
- source and work-contract identifiers
|
|
55
|
-
- a monotonic sequence number
|
|
56
|
-
- job state and current phase
|
|
57
|
-
- start, update, optional deadline, and optional finish timestamps
|
|
58
|
-
- one or more keyed `pi-workflows.progress.v1` tracks
|
|
59
|
-
- optional settled cost and active reservation facts
|
|
60
|
-
|
|
61
|
-
The validator is strict. It rejects unknown fields, duplicate track keys, invalid timestamps, non-finite values, invalid progress tracks, and oversized snapshots.
|
|
62
|
-
|
|
63
|
-
The reporter accepts an injected asynchronous `publish(snapshot)` callback. Pi Workflows does not import a Hugging Face client. The reporter:
|
|
64
|
-
|
|
65
|
-
- keeps sequence numbers monotonic within the process
|
|
66
|
-
- rejects regressions within one phase and epoch
|
|
67
|
-
- coalesces frequent updates with a configurable minimum interval
|
|
68
|
-
- flushes phase changes and terminal states immediately
|
|
69
|
-
- bounds publication time with an abort deadline
|
|
70
|
-
- preserves the most recent unsent snapshot after a transient publication failure
|
|
71
|
-
- never includes arbitrary metadata or environment values
|
|
72
|
-
|
|
73
|
-
A terminal snapshot is operational evidence only. The application's receipt and durable output validation still decide whether work succeeded.
|
|
74
|
-
|
|
75
|
-
## Storage and discovery
|
|
76
|
-
|
|
77
|
-
Each application writes snapshots to its existing Bucket.
|
|
78
|
-
|
|
79
|
-
xTap Pool uses:
|
|
80
|
-
|
|
81
|
-
```text
|
|
82
|
-
osolmaz/xtap-pool-bucket/operations/enrichment/runs/<job-id>/progress.json
|
|
83
|
-
```
|
|
84
|
-
|
|
85
|
-
OurModels uses:
|
|
86
|
-
|
|
87
|
-
```text
|
|
88
|
-
osolmaz/ourmodels-data/<prefix>/operations/community-posts/runs/<job-id>/progress.json
|
|
89
|
-
```
|
|
90
|
-
|
|
91
|
-
Each schedule supplies these immutable labels:
|
|
92
|
-
|
|
93
|
-
```text
|
|
94
|
-
progress_schema=pi-workflows.job-progress.v1
|
|
95
|
-
progress_bucket=<bucket>
|
|
96
|
-
progress_prefix=<path-before-job-id>
|
|
97
|
-
```
|
|
98
|
-
|
|
99
|
-
A monitor reads the labels, appends the physical job identifier and `progress.json`, reads the snapshot with existing local Hugging Face authentication, validates it, and publishes each track under a stable workflow progress key. The monitor does not trust an ETA string from logs. It uses source finish time when the snapshot provides one, or the existing conservative estimator after enough measured samples.
|
|
100
|
-
|
|
101
|
-
## xTap Pool tracks
|
|
102
|
-
|
|
103
|
-
The enrichment worker reports these stable tracks when facts are measurable:
|
|
104
|
-
|
|
105
|
-
| Key | Unit | Source |
|
|
106
|
-
| ------------------ | ---------- | -------------------------------------------------- |
|
|
107
|
-
| `database-restore` | bytes | downloaded database bytes and expected object size |
|
|
108
|
-
| `registry-replay` | events | replayed registry events and discovered total |
|
|
109
|
-
| `registry-scan` | candidates | durable scan cursor and fixed candidate total |
|
|
110
|
-
| `queue` | records | terminal records and durable queue total |
|
|
111
|
-
| `publication` | bytes | uploaded and verified database bytes |
|
|
112
|
-
| `overall` | phases | completed phases and fixed phase count |
|
|
113
|
-
|
|
114
|
-
Phase-only states remain valid when a total is not yet known. The worker must not invent totals.
|
|
115
|
-
|
|
116
|
-
The existing three unresolved records must receive durable outcomes under the fixed full-response deadline or become exactly validated blocked records. The worker then publishes and verifies the index before its final receipt is accepted.
|
|
117
|
-
|
|
118
|
-
## OurModels tracks
|
|
119
|
-
|
|
120
|
-
The community-posts worker reports these stable tracks when facts are measurable:
|
|
121
|
-
|
|
122
|
-
| Key | Unit | Source |
|
|
123
|
-
| ----------------- | ------- | ----------------------------------------------- |
|
|
124
|
-
| `model-discovery` | models | discovered and inspected model count |
|
|
125
|
-
| `community-posts` | models | processed models and fixed discovered total |
|
|
126
|
-
| `cache` | records | durable cached model results and expected total |
|
|
127
|
-
| `publication` | bytes | uploaded and verified artifact bytes |
|
|
128
|
-
| `overall` | phases | completed phases and fixed phase count |
|
|
129
|
-
|
|
130
|
-
The worker continues to use its existing receipt and manifest rules. A progress write failure must not corrupt a useful checkpoint or published result.
|
|
131
|
-
|
|
132
|
-
## Delivery sequence
|
|
133
|
-
|
|
134
|
-
1. Add and release `@osolmaz/pi-workflows/job-progress` as version `0.8.0`.
|
|
135
|
-
2. Add snapshot discovery guidance to the bundled monitor skill.
|
|
136
|
-
3. Update xTap Pool to use `0.8.0`, add measured callbacks, and add schedule labels.
|
|
137
|
-
4. Merge and deploy xTap Pool, then replace the old suspended schedule.
|
|
138
|
-
5. End the old physical job only when the replacement source is ready and the one-job rule can be preserved.
|
|
139
|
-
6. Start one instrumented xTap Pool recovery job under the existing restoration budget.
|
|
140
|
-
7. Update OurModels to use `0.8.0`, add measured callbacks, and replace old schedules with one suspended instrumented schedule.
|
|
141
|
-
8. Do not start a paid OurModels run until its measured cost range and ceiling receive the required approval.
|
|
142
|
-
9. Start a Pi monitor that reads both progress surfaces and displays current progress and ETA.
|
|
143
|
-
|
|
144
|
-
## Acceptance checks
|
|
145
|
-
|
|
146
|
-
Pi Workflows must pass:
|
|
147
|
-
|
|
148
|
-
```bash
|
|
149
|
-
npm run check
|
|
150
|
-
npm run test:e2e
|
|
151
|
-
npx slophammer-ts@latest dry .
|
|
152
|
-
npx slophammer-ts@latest check . --only ts.dependency-boundaries-required
|
|
153
|
-
```
|
|
154
|
-
|
|
155
|
-
Tests must cover strict validation, unknown fields, duplicate keys, monotonic updates, phase resets, coalescing, transient publication failure, deadline abort, terminal flush, and ETA estimation from snapshots.
|
|
156
|
-
|
|
157
|
-
xTap Pool must pass its repository checks, including mutation testing. A live recovery must show a valid snapshot in the existing index Bucket, and repeated monitor samples must produce a measured ETA when a track has a known total and positive progress.
|
|
158
|
-
|
|
159
|
-
OurModels must pass:
|
|
160
|
-
|
|
161
|
-
```bash
|
|
162
|
-
npm run check
|
|
163
|
-
npm run coverage
|
|
164
|
-
npm run dry
|
|
165
|
-
npm run mutate
|
|
166
|
-
node scripts/test-bounds-engine.mjs
|
|
167
|
-
node scripts/validate-model-data.mjs
|
|
168
|
-
```
|
|
169
|
-
|
|
170
|
-
Its replacement schedule must remain suspended with only the approved two secrets until a paid run is authorized.
|
|
171
|
-
|
|
172
|
-
## Recovery
|
|
173
|
-
|
|
174
|
-
If a progress publication fails, the worker keeps useful work and retries only the latest snapshot at the next bounded update. If the job ends before that succeeds, the receipt and durable outputs remain authoritative and the monitor marks progress stale.
|
|
175
|
-
|
|
176
|
-
If a repeated deterministic worker defect appears, stop the affected job and schedule. Preserve all existing Bucket objects and return the exact failing phase, source revision, snapshot, receipt state, and last valid durable outputs.
|
|
@@ -1,24 +0,0 @@
|
|
|
1
|
-
export {
|
|
2
|
-
createJobProgressReporter,
|
|
3
|
-
estimateJobProgress,
|
|
4
|
-
type JobProgressReporter,
|
|
5
|
-
} from "./reporter.js";
|
|
6
|
-
export {
|
|
7
|
-
isTerminalJobProgressState,
|
|
8
|
-
MAX_JOB_PROGRESS_BYTES,
|
|
9
|
-
MAX_JOB_PROGRESS_TRACKS,
|
|
10
|
-
validateJobProgressSnapshot,
|
|
11
|
-
} from "./validation.js";
|
|
12
|
-
export {
|
|
13
|
-
JOB_PROGRESS_SCHEMA,
|
|
14
|
-
type JobProgressCost,
|
|
15
|
-
type JobProgressEstimate,
|
|
16
|
-
type JobProgressIdentity,
|
|
17
|
-
type JobProgressPublish,
|
|
18
|
-
type JobProgressPublishResult,
|
|
19
|
-
type JobProgressReporterOptions,
|
|
20
|
-
type JobProgressSnapshot,
|
|
21
|
-
type JobProgressState,
|
|
22
|
-
type JobProgressTrack,
|
|
23
|
-
type JobProgressUpdate,
|
|
24
|
-
} from "./types.js";
|