@osolmaz/pi-workflows 0.8.0 → 0.8.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -1,65 +0,0 @@
1
- import type { ProgressEstimate, ProgressTrackState, WorkflowProgressData } from "../workflows/index.js";
2
- export declare const JOB_PROGRESS_SCHEMA: "pi-workflows.job-progress.v1";
3
- export type JobProgressState = "queued" | "running" | "waiting" | "blocked" | "completed" | "failed" | "cancelled" | "unknown";
4
- export type JobProgressTrack = {
5
- key: string;
6
- data: WorkflowProgressData;
7
- };
8
- export type JobProgressCost = {
9
- settledUsd: number;
10
- reservedUsd: number;
11
- };
12
- export type JobProgressSnapshot = {
13
- schema: typeof JOB_PROGRESS_SCHEMA;
14
- application: string;
15
- component: string;
16
- jobId: string;
17
- sourceRevision: string;
18
- contractHash: string;
19
- sequence: number;
20
- state: JobProgressState;
21
- phase: string;
22
- startedAt: string;
23
- updatedAt: string;
24
- deadlineAt?: string;
25
- finishedAt?: string;
26
- tracks: JobProgressTrack[];
27
- cost?: JobProgressCost;
28
- };
29
- export type JobProgressIdentity = Pick<JobProgressSnapshot, "application" | "component" | "jobId" | "sourceRevision" | "contractHash" | "startedAt"> & {
30
- deadlineAt?: string;
31
- };
32
- export type JobProgressUpdate = {
33
- state?: JobProgressState;
34
- phase?: string;
35
- tracks?: JobProgressTrack[];
36
- cost?: JobProgressCost;
37
- finishedAt?: string;
38
- force?: boolean;
39
- };
40
- export type JobProgressPublish = (snapshot: JobProgressSnapshot, signal: AbortSignal) => Promise<void>;
41
- type JobProgressReporterBaseOptions = JobProgressIdentity & {
42
- initialState?: Exclude<JobProgressState, "completed" | "failed" | "cancelled">;
43
- initialPhase?: string;
44
- publish: JobProgressPublish;
45
- minimumIntervalMs?: number;
46
- publishTimeoutMs?: number;
47
- now?: () => Date;
48
- };
49
- export type JobProgressReporterOptions = JobProgressReporterBaseOptions & ({
50
- initialTracks: JobProgressTrack[];
51
- previousSnapshot?: never;
52
- } | {
53
- initialTracks?: JobProgressTrack[];
54
- previousSnapshot: JobProgressSnapshot;
55
- });
56
- export type JobProgressPublishResult = {
57
- snapshot: JobProgressSnapshot;
58
- published: boolean;
59
- };
60
- export type JobProgressEstimate = {
61
- snapshot: JobProgressSnapshot;
62
- tracks: ProgressTrackState[];
63
- estimates: ProgressEstimate[];
64
- };
65
- export {};
@@ -1,2 +0,0 @@
1
- export const JOB_PROGRESS_SCHEMA = "pi-workflows.job-progress.v1";
2
- //# sourceMappingURL=types.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"types.js","sourceRoot":"","sources":["../../src/job-progress/types.ts"],"names":[],"mappings":"AAMA,MAAM,CAAC,MAAM,mBAAmB,GAAG,8BAAuC,CAAC"}
@@ -1,5 +0,0 @@
1
- import { type JobProgressSnapshot, type JobProgressState } from "./types.js";
2
- export declare const MAX_JOB_PROGRESS_BYTES: number;
3
- export declare const MAX_JOB_PROGRESS_TRACKS = 128;
4
- export declare function validateJobProgressSnapshot(value: unknown): JobProgressSnapshot;
5
- export declare function isTerminalJobProgressState(state: JobProgressState): state is "completed" | "failed" | "cancelled";
@@ -1,227 +0,0 @@
1
- import { validateProgressData } from "../workflows/index.js";
2
- import { JOB_PROGRESS_SCHEMA, } from "./types.js";
3
- export const MAX_JOB_PROGRESS_BYTES = 64 * 1024;
4
- export const MAX_JOB_PROGRESS_TRACKS = 128;
5
- const SNAPSHOT_FIELDS = new Set([
6
- "schema",
7
- "application",
8
- "component",
9
- "jobId",
10
- "sourceRevision",
11
- "contractHash",
12
- "sequence",
13
- "state",
14
- "phase",
15
- "startedAt",
16
- "updatedAt",
17
- "deadlineAt",
18
- "finishedAt",
19
- "tracks",
20
- "cost",
21
- ]);
22
- const TRACK_FIELDS = new Set(["key", "data"]);
23
- const COST_FIELDS = new Set(["settledUsd", "reservedUsd"]);
24
- const STATES = new Set([
25
- "queued",
26
- "running",
27
- "waiting",
28
- "blocked",
29
- "completed",
30
- "failed",
31
- "cancelled",
32
- "unknown",
33
- ]);
34
- const KEY_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}$/;
35
- const RFC_3339_PATTERN = /^(\d{4})-(\d{2})-(\d{2})T(?:[01]\d|2[0-3]):[0-5]\d:[0-5]\d(?:\.\d+)?(?:Z|[+-](?:[01]\d|2[0-3]):[0-5]\d)$/;
36
- export function validateJobProgressSnapshot(value) {
37
- const bytes = jsonBytes(value);
38
- if (bytes > MAX_JOB_PROGRESS_BYTES) {
39
- throw new Error(`job progress snapshot must be at most ${MAX_JOB_PROGRESS_BYTES} bytes`);
40
- }
41
- if (!isRecord(value))
42
- throw new Error("job progress snapshot must be an object");
43
- rejectUnknown(value, SNAPSHOT_FIELDS, "job progress");
44
- if (value.schema !== JOB_PROGRESS_SCHEMA) {
45
- throw new Error(`job progress.schema must equal ${JOB_PROGRESS_SCHEMA}`);
46
- }
47
- const application = requiredText(value.application, "job progress.application", 100);
48
- const component = requiredText(value.component, "job progress.component", 100);
49
- const jobId = requiredText(value.jobId, "job progress.jobId", 200);
50
- const sourceRevision = requiredText(value.sourceRevision, "job progress.sourceRevision", 200);
51
- const contractHash = requiredText(value.contractHash, "job progress.contractHash", 200);
52
- const sequence = requiredInteger(value.sequence, "job progress.sequence");
53
- const state = requiredState(value.state);
54
- const phase = requiredText(value.phase, "job progress.phase", 100);
55
- const startedAt = requiredTimestamp(value.startedAt, "job progress.startedAt");
56
- const updatedAt = requiredTimestamp(value.updatedAt, "job progress.updatedAt");
57
- const deadlineAt = optionalTimestamp(value.deadlineAt, "job progress.deadlineAt");
58
- const finishedAt = optionalTimestamp(value.finishedAt, "job progress.finishedAt");
59
- const tracks = requiredTracks(value.tracks);
60
- const cost = optionalCost(value.cost);
61
- if (Date.parse(updatedAt) < Date.parse(startedAt)) {
62
- throw new Error("job progress.updatedAt must not be before startedAt");
63
- }
64
- if (deadlineAt !== undefined && Date.parse(deadlineAt) <= Date.parse(startedAt)) {
65
- throw new Error("job progress.deadlineAt must be after startedAt");
66
- }
67
- if (finishedAt !== undefined && Date.parse(finishedAt) < Date.parse(startedAt)) {
68
- throw new Error("job progress.finishedAt must not be before startedAt");
69
- }
70
- if (finishedAt !== undefined && Date.parse(finishedAt) > Date.parse(updatedAt)) {
71
- throw new Error("job progress.finishedAt must not be after updatedAt");
72
- }
73
- if (isTerminal(state) && finishedAt === undefined) {
74
- throw new Error("job progress.finishedAt is required for a terminal state");
75
- }
76
- if (!isTerminal(state) && finishedAt !== undefined) {
77
- throw new Error("job progress.finishedAt is allowed only for a terminal state");
78
- }
79
- if (isTerminal(state) && tracks.some((track) => !isTerminalTrack(track))) {
80
- throw new Error("job progress tracks must be terminal when the job state is terminal");
81
- }
82
- return {
83
- schema: JOB_PROGRESS_SCHEMA,
84
- application,
85
- component,
86
- jobId,
87
- sourceRevision,
88
- contractHash,
89
- sequence,
90
- state,
91
- phase,
92
- startedAt,
93
- updatedAt,
94
- ...(deadlineAt === undefined ? {} : { deadlineAt }),
95
- ...(finishedAt === undefined ? {} : { finishedAt }),
96
- tracks,
97
- ...(cost === undefined ? {} : { cost }),
98
- };
99
- }
100
- export function isTerminalJobProgressState(state) {
101
- return isTerminal(state);
102
- }
103
- function requiredTracks(value) {
104
- if (!Array.isArray(value))
105
- throw new Error("job progress.tracks must be an array");
106
- if (value.length < 1)
107
- throw new Error("job progress.tracks must contain at least one entry");
108
- if (value.length > MAX_JOB_PROGRESS_TRACKS) {
109
- throw new Error(`job progress.tracks must contain at most ${MAX_JOB_PROGRESS_TRACKS} entries`);
110
- }
111
- const keys = new Set();
112
- return value.map((item, index) => {
113
- if (!isRecord(item))
114
- throw new Error(`job progress.tracks[${index}] must be an object`);
115
- rejectUnknown(item, TRACK_FIELDS, `job progress.tracks[${index}]`);
116
- if (typeof item.key !== "string" || !KEY_PATTERN.test(item.key)) {
117
- throw new Error(`job progress.tracks[${index}].key must match [A-Za-z0-9][A-Za-z0-9._:/-]{0,127}`);
118
- }
119
- if (keys.has(item.key))
120
- throw new Error(`job progress track key ${item.key} is duplicated`);
121
- keys.add(item.key);
122
- if (!isRecord(item.data)) {
123
- throw new Error(`job progress.tracks[${index}].data must be an object`);
124
- }
125
- return { key: item.key, data: structuredClone(validateProgressData(item.data)) };
126
- });
127
- }
128
- function optionalCost(value) {
129
- if (value === undefined)
130
- return undefined;
131
- if (!isRecord(value))
132
- throw new Error("job progress.cost must be an object");
133
- rejectUnknown(value, COST_FIELDS, "job progress.cost");
134
- return {
135
- settledUsd: requiredNonNegative(value.settledUsd, "job progress.cost.settledUsd"),
136
- reservedUsd: requiredNonNegative(value.reservedUsd, "job progress.cost.reservedUsd"),
137
- };
138
- }
139
- function requiredState(value) {
140
- if (typeof value !== "string" || !STATES.has(value)) {
141
- throw new Error("job progress.state is invalid");
142
- }
143
- return value;
144
- }
145
- function requiredText(value, field, max) {
146
- if (typeof value !== "string" || value.trim().length < 1 || value.trim().length > max) {
147
- throw new Error(`${field} must be 1 to ${max} characters`);
148
- }
149
- if ([...value].some(isControlCharacter))
150
- throw new Error(`${field} must not contain control characters`);
151
- return value;
152
- }
153
- function requiredInteger(value, field) {
154
- if (!Number.isSafeInteger(value) || value < 0) {
155
- throw new Error(`${field} must be a non-negative safe integer`);
156
- }
157
- return value;
158
- }
159
- function requiredNonNegative(value, field) {
160
- if (typeof value !== "number" || !Number.isFinite(value) || value < 0) {
161
- throw new Error(`${field} must be a finite non-negative number`);
162
- }
163
- return value;
164
- }
165
- function requiredTimestamp(value, field) {
166
- const parsed = optionalTimestamp(value, field);
167
- if (parsed === undefined)
168
- throw new Error(`${field} is required`);
169
- return parsed;
170
- }
171
- function optionalTimestamp(value, field) {
172
- if (value === undefined)
173
- return undefined;
174
- if (typeof value !== "string") {
175
- throw new Error(`${field} must be an RFC 3339 timestamp with an offset`);
176
- }
177
- const match = RFC_3339_PATTERN.exec(value);
178
- if (match === null || !validCalendarDate(match[1], match[2], match[3])) {
179
- throw new Error(`${field} must be an RFC 3339 timestamp with an offset`);
180
- }
181
- return value;
182
- }
183
- function validCalendarDate(yearText, monthText, dayText) {
184
- const year = Number(yearText);
185
- const month = Number(monthText);
186
- const day = Number(dayText);
187
- if (!Number.isInteger(year) || year < 1 || month < 1 || month > 12 || day < 1)
188
- return false;
189
- return day <= new Date(Date.UTC(year, month, 0)).getUTCDate();
190
- }
191
- function rejectUnknown(value, allowed, field) {
192
- for (const key of Object.keys(value)) {
193
- if (!allowed.has(key))
194
- throw new Error(`${field}.${key} is not supported`);
195
- }
196
- }
197
- function isRecord(value) {
198
- if (value === null || typeof value !== "object" || Array.isArray(value))
199
- return false;
200
- const prototype = Object.getPrototypeOf(value);
201
- return prototype === Object.prototype || prototype === null;
202
- }
203
- function jsonBytes(value) {
204
- let encoded;
205
- try {
206
- encoded = JSON.stringify(value);
207
- }
208
- catch {
209
- throw new Error("job progress snapshot must be JSON serializable");
210
- }
211
- if (encoded === undefined)
212
- throw new Error("job progress snapshot must be JSON serializable");
213
- return Buffer.byteLength(encoded, "utf8");
214
- }
215
- function isControlCharacter(character) {
216
- const code = character.codePointAt(0) ?? 0;
217
- return code < 32 || (code >= 127 && code <= 159);
218
- }
219
- function isTerminal(state) {
220
- return state === "completed" || state === "failed" || state === "cancelled";
221
- }
222
- function isTerminalTrack(track) {
223
- return (track.data.status === "completed" ||
224
- track.data.status === "failed" ||
225
- track.data.status === "cancelled");
226
- }
227
- //# sourceMappingURL=validation.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"validation.js","sourceRoot":"","sources":["../../src/job-progress/validation.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,oBAAoB,EAAE,MAAM,uBAAuB,CAAC;AAC7D,OAAO,EACL,mBAAmB,GAKpB,MAAM,YAAY,CAAC;AAEpB,MAAM,CAAC,MAAM,sBAAsB,GAAG,EAAE,GAAG,IAAI,CAAC;AAChD,MAAM,CAAC,MAAM,uBAAuB,GAAG,GAAG,CAAC;AAE3C,MAAM,eAAe,GAAG,IAAI,GAAG,CAAC;IAC9B,QAAQ;IACR,aAAa;IACb,WAAW;IACX,OAAO;IACP,gBAAgB;IAChB,cAAc;IACd,UAAU;IACV,OAAO;IACP,OAAO;IACP,WAAW;IACX,WAAW;IACX,YAAY;IACZ,YAAY;IACZ,QAAQ;IACR,MAAM;CACP,CAAC,CAAC;AACH,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC,CAAC,KAAK,EAAE,MAAM,CAAC,CAAC,CAAC;AAC9C,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,YAAY,EAAE,aAAa,CAAC,CAAC,CAAC;AAC3D,MAAM,MAAM,GAAG,IAAI,GAAG,CAAmB;IACvC,QAAQ;IACR,SAAS;IACT,SAAS;IACT,SAAS;IACT,WAAW;IACX,QAAQ;IACR,WAAW;IACX,SAAS;CACV,CAAC,CAAC;AACH,MAAM,WAAW,GAAG,sCAAsC,CAAC;AAC3D,MAAM,gBAAgB,GACpB,0GAA0G,CAAC;AAE7G,MAAM,UAAU,2BAA2B,CAAC,KAAc;IACxD,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,IAAI,KAAK,GAAG,sBAAsB,EAAE,CAAC;QACnC,MAAM,IAAI,KAAK,CAAC,yCAAyC,sBAAsB,QAAQ,CAAC,CAAC;IAC3F,CAAC;IACD,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,yCAAyC,CAAC,CAAC;IACjF,aAAa,CAAC,KAAK,EAAE,eAAe,EAAE,cAAc,CAAC,CAAC;IACtD,IAAI,KAAK,CAAC,MAAM,KAAK,mBAAmB,EAAE,CAAC;QACzC,MAAM,IAAI,KAAK,CAAC,kCAAkC,mBAAmB,EAAE,CAAC,CAAC;IAC3E,CAAC;IAED,MAAM,WAAW,GAAG,YAAY,CAAC,KAAK,CAAC,WAAW,EAAE,0BAA0B,EAAE,GAAG,CAAC,CAAC;IACrF,MAAM,SAAS,GAAG,YAAY,CAAC,KAAK,CAAC,SAAS,EAAE,wBAAwB,EAAE,GAAG,CAAC,CAAC;IAC/E,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,oBAAoB,EAAE,GAAG,CAAC,CAAC;IACnE,MAAM,cAAc,GAAG,YAAY,CAAC,KAAK,CAAC,cAAc,EAAE,6BAA6B,EAAE,GAAG,CAAC,CAAC;IAC9F,MAAM,YAAY,GAAG,YAAY,CAAC,KAAK,CAAC,YAAY,EAAE,2BAA2B,EAAE,GAAG,CAAC,CAAC;IACxF,MAAM,QAAQ,GAAG,eAAe,CAAC,KAAK,CAAC,QAAQ,EAAE,uBAAuB,CAAC,CAAC;IAC1E,MAAM,KAAK,GAAG,aAAa,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,YAAY,CAAC,KAAK,CAAC,KAAK,EAAE,oBAAoB,EAAE,GAAG,CAAC,CAAC;IACnE,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;IAC/E,MAAM,SAAS,GAAG,iBAAiB,CAAC,KAAK,CAAC,SAAS,EAAE,wBAAwB,CAAC,CAAC;IAC/E,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAC;IAClF,MAAM,UAAU,GAAG,iBAAiB,CAAC,KAAK,CAAC,UAAU,EAAE,yBAAyB,CAAC,CAAC;IAClF,MAAM,MAAM,GAAG,cAAc,CAAC,KAAK,CAAC,MAAM,CAAC,CAAC;IAC5C,MAAM,IAAI,GAAG,YAAY,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAEtC,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,IAAI,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAChF,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/E,MAAM,IAAI,KAAK,CAAC,sDAAsD,CAAC,CAAC;IAC1E,CAAC;IACD,IAAI,UAAU,KAAK,SAAS,IAAI,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC;QAC/E,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACzE,CAAC;IACD,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QAClD,MAAM,IAAI,KAAK,CAAC,0DAA0D,CAAC,CAAC;IAC9E,CAAC;IACD,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,UAAU,KAAK,SAAS,EAAE,CAAC;QACnD,MAAM,IAAI,KAAK,CAAC,8DAA8D,CAAC,CAAC;IAClF,CAAC;IACD,IAAI,UAAU,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC,eAAe,CAAC,KAAK,CAAC,CAAC,EAAE,CAAC;QACzE,MAAM,IAAI,KAAK,CAAC,qEAAqE,CAAC,CAAC;IACzF,CAAC;IAED,OAAO;QACL,MAAM,EAAE,mBAAmB;QAC3B,WAAW;QACX,SAAS;QACT,KAAK;QACL,cAAc;QACd,YAAY;QACZ,QAAQ;QACR,KAAK;QACL,KAAK;QACL,SAAS;QACT,SAAS;QACT,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;QACnD,GAAG,CAAC,UAAU,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,UAAU,EAAE,CAAC;QACnD,MAAM;QACN,GAAG,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,CAAC;KACxC,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,0BAA0B,CACxC,KAAuB;IAEvB,OAAO,UAAU,CAAC,KAAK,CAAC,CAAC;AAC3B,CAAC;AAED,SAAS,cAAc,CAAC,KAAc;IACpC,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,sCAAsC,CAAC,CAAC;IACnF,IAAI,KAAK,CAAC,MAAM,GAAG,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IAC7F,IAAI,KAAK,CAAC,MAAM,GAAG,uBAAuB,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,4CAA4C,uBAAuB,UAAU,CAAC,CAAC;IACjG,CAAC;IACD,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,OAAO,KAAK,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QAC/B,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,uBAAuB,KAAK,qBAAqB,CAAC,CAAC;QACxF,aAAa,CAAC,IAAI,EAAE,YAAY,EAAE,uBAAuB,KAAK,GAAG,CAAC,CAAC;QACnE,IAAI,OAAO,IAAI,CAAC,GAAG,KAAK,QAAQ,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;YAChE,MAAM,IAAI,KAAK,CACb,uBAAuB,KAAK,qDAAqD,CAClF,CAAC;QACJ,CAAC;QACD,IAAI,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,0BAA0B,IAAI,CAAC,GAAG,gBAAgB,CAAC,CAAC;QAC5F,IAAI,CAAC,GAAG,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;QACnB,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC;YACzB,MAAM,IAAI,KAAK,CAAC,uBAAuB,KAAK,0BAA0B,CAAC,CAAC;QAC1E,CAAC;QACD,OAAO,EAAE,GAAG,EAAE,IAAI,CAAC,GAAG,EAAE,IAAI,EAAE,eAAe,CAAC,oBAAoB,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;IACnF,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,YAAY,CAAC,KAAc;IAClC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,MAAM,IAAI,KAAK,CAAC,qCAAqC,CAAC,CAAC;IAC7E,aAAa,CAAC,KAAK,EAAE,WAAW,EAAE,mBAAmB,CAAC,CAAC;IACvD,OAAO;QACL,UAAU,EAAE,mBAAmB,CAAC,KAAK,CAAC,UAAU,EAAE,8BAA8B,CAAC;QACjF,WAAW,EAAE,mBAAmB,CAAC,KAAK,CAAC,WAAW,EAAE,+BAA+B,CAAC;KACrF,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,GAAG,CAAC,KAAyB,CAAC,EAAE,CAAC;QACxE,MAAM,IAAI,KAAK,CAAC,+BAA+B,CAAC,CAAC;IACnD,CAAC;IACD,OAAO,KAAyB,CAAC;AACnC,CAAC;AAED,SAAS,YAAY,CAAC,KAAc,EAAE,KAAa,EAAE,GAAW;IAC9D,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;QACtF,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,iBAAiB,GAAG,aAAa,CAAC,CAAC;IAC7D,CAAC;IACD,IAAI,CAAC,GAAG,KAAK,CAAC,CAAC,IAAI,CAAC,kBAAkB,CAAC;QACrC,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,sCAAsC,CAAC,CAAC;IAClE,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,KAAa;IACpD,IAAI,CAAC,MAAM,CAAC,aAAa,CAAC,KAAK,CAAC,IAAK,KAAgB,GAAG,CAAC,EAAE,CAAC;QAC1D,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,sCAAsC,CAAC,CAAC;IAClE,CAAC;IACD,OAAO,KAAe,CAAC;AACzB,CAAC;AAED,SAAS,mBAAmB,CAAC,KAAc,EAAE,KAAa;IACxD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,KAAK,GAAG,CAAC,EAAE,CAAC;QACtE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,uCAAuC,CAAC,CAAC;IACnE,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAE,KAAa;IACtD,MAAM,MAAM,GAAG,iBAAiB,CAAC,KAAK,EAAE,KAAK,CAAC,CAAC;IAC/C,IAAI,MAAM,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,cAAc,CAAC,CAAC;IAClE,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,SAAS,iBAAiB,CAAC,KAAc,EAAE,KAAa;IACtD,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC1C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,+CAA+C,CAAC,CAAC;IAC3E,CAAC;IACD,MAAM,KAAK,GAAG,gBAAgB,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC3C,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,iBAAiB,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;QACvE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,+CAA+C,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,SAAS,iBAAiB,CACxB,QAA4B,EAC5B,SAA6B,EAC7B,OAA2B;IAE3B,MAAM,IAAI,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC9B,MAAM,KAAK,GAAG,MAAM,CAAC,SAAS,CAAC,CAAC;IAChC,MAAM,GAAG,GAAG,MAAM,CAAC,OAAO,CAAC,CAAC;IAC5B,IAAI,CAAC,MAAM,CAAC,SAAS,CAAC,IAAI,CAAC,IAAI,IAAI,GAAG,CAAC,IAAI,KAAK,GAAG,CAAC,IAAI,KAAK,GAAG,EAAE,IAAI,GAAG,GAAG,CAAC;QAAE,OAAO,KAAK,CAAC;IAC5F,OAAO,GAAG,IAAI,IAAI,IAAI,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,EAAE,CAAC;AAChE,CAAC;AAED,SAAS,aAAa,CACpB,KAA8B,EAC9B,OAA4B,EAC5B,KAAa;IAEb,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,GAAG,CAAC;YAAE,MAAM,IAAI,KAAK,CAAC,GAAG,KAAK,IAAI,GAAG,mBAAmB,CAAC,CAAC;IAC7E,CAAC;AACH,CAAC;AAED,SAAS,QAAQ,CAAC,KAAc;IAC9B,IAAI,KAAK,KAAK,IAAI,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC;QAAE,OAAO,KAAK,CAAC;IACtF,MAAM,SAAS,GAAG,MAAM,CAAC,cAAc,CAAC,KAAK,CAAY,CAAC;IAC1D,OAAO,SAAS,KAAK,MAAM,CAAC,SAAS,IAAI,SAAS,KAAK,IAAI,CAAC;AAC9D,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,OAA2B,CAAC;IAChC,IAAI,CAAC;QACH,OAAO,GAAG,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAClC,CAAC;IAAC,MAAM,CAAC;QACP,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IACrE,CAAC;IACD,IAAI,OAAO,KAAK,SAAS;QAAE,MAAM,IAAI,KAAK,CAAC,iDAAiD,CAAC,CAAC;IAC9F,OAAO,MAAM,CAAC,UAAU,CAAC,OAAO,EAAE,MAAM,CAAC,CAAC;AAC5C,CAAC;AAED,SAAS,kBAAkB,CAAC,SAAiB;IAC3C,MAAM,IAAI,GAAG,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC;IAC3C,OAAO,IAAI,GAAG,EAAE,IAAI,CAAC,IAAI,IAAI,GAAG,IAAI,IAAI,IAAI,GAAG,CAAC,CAAC;AACnD,CAAC;AAED,SAAS,UAAU,CAAC,KAAuB;IACzC,OAAO,KAAK,KAAK,WAAW,IAAI,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,WAAW,CAAC;AAC9E,CAAC;AAED,SAAS,eAAe,CAAC,KAAuB;IAC9C,OAAO,CACL,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW;QACjC,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,QAAQ;QAC9B,KAAK,CAAC,IAAI,CAAC,MAAM,KAAK,WAAW,CAClC,CAAC;AACJ,CAAC"}
@@ -1,119 +0,0 @@
1
- # Durable job progress
2
-
3
- `@osolmaz/pi-workflows/job-progress` lets a remote job publish progress that a Pi Workflows monitor can validate and measure. It uses the existing `pi-workflows.progress.v1` track contract and ETA estimator.
4
-
5
- ## Report progress
6
-
7
- Create one reporter for one physical job. Inject the storage write so the package remains independent of a cloud provider.
8
-
9
- ```ts
10
- import { createJobProgressReporter } from "@osolmaz/pi-workflows/job-progress";
11
-
12
- const reporter = createJobProgressReporter({
13
- application: "example",
14
- component: "batch-worker",
15
- jobId: process.env.JOB_ID ?? "local",
16
- sourceRevision: "abc123",
17
- contractHash: "def456",
18
- startedAt: new Date().toISOString(),
19
- initialTracks: [
20
- {
21
- key: "overall",
22
- data: {
23
- schema: "pi-workflows.progress.v1",
24
- status: "running",
25
- phase: "starting",
26
- },
27
- },
28
- ],
29
- minimumIntervalMs: 30_000,
30
- publishTimeoutMs: 15_000,
31
- publish: async (snapshot, signal) => {
32
- await bucket.writeText(progressPath, JSON.stringify(snapshot), { signal });
33
- },
34
- });
35
-
36
- await reporter.report({
37
- phase: "processing",
38
- tracks: [
39
- {
40
- key: "records",
41
- data: {
42
- schema: "pi-workflows.progress.v1",
43
- status: "running",
44
- phase: "processing",
45
- completed: 400,
46
- total: 1_000,
47
- unit: "records",
48
- },
49
- },
50
- ],
51
- });
52
- ```
53
-
54
- The first update, each phase change, and each terminal update publishes immediately. Other updates are coalesced until `minimumIntervalMs` has passed. Call `flush()` at a durable checkpoint or before exit when the current snapshot has not been published.
55
-
56
- On process restart, read and validate the existing snapshot and pass it as `previousSnapshot`. The reporter continues its sequence number and rejects an identity, deadline, or terminal-state mismatch.
57
-
58
- The reporter keeps the latest snapshot after a write failure. The application decides when to log and retry that failure. A timed-out storage write remains serialized until the underlying write settles, so an older write cannot overwrite a newer snapshot. Storage adapters must honor the abort signal and settle after cancellation. A progress failure must not replace receipt or checkpoint validation.
59
-
60
- ## Finish a job
61
-
62
- A terminal update gets a finish timestamp and publishes immediately:
63
-
64
- ```ts
65
- await reporter.report({
66
- state: "completed",
67
- phase: "complete",
68
- tracks: [
69
- {
70
- key: "records",
71
- data: {
72
- schema: "pi-workflows.progress.v1",
73
- status: "completed",
74
- phase: "complete",
75
- completed: 1_000,
76
- total: 1_000,
77
- unit: "records",
78
- },
79
- },
80
- ],
81
- });
82
- ```
83
-
84
- A terminal snapshot is not a receipt. Receipts, manifests, hashes, and durable application outputs remain authoritative.
85
-
86
- ## Store and discover snapshots
87
-
88
- Write one mutable snapshot per physical job:
89
-
90
- ```text
91
- <existing-bucket>/<application-prefix>/runs/<job-id>/progress.json
92
- ```
93
-
94
- Add these immutable labels to the job or schedule:
95
-
96
- ```text
97
- progress_schema=pi-workflows.job-progress.v1
98
- progress_bucket=<existing-bucket>
99
- progress_prefix=<application-prefix>/runs
100
- ```
101
-
102
- A monitor reads the labels, forms `<progress_prefix>/<job-id>/progress.json`, validates the snapshot with `validateJobProgressSnapshot`, and publishes its tracks with stable keys. It should retain consecutive snapshots so Pi Workflows can estimate a conservative ETA from measured rates.
103
-
104
- ## Estimate from snapshots
105
-
106
- ```ts
107
- import { estimateJobProgress } from "@osolmaz/pi-workflows/job-progress";
108
-
109
- const result = estimateJobProgress(previousSnapshots);
110
- for (const estimate of result.estimates) {
111
- console.log(estimate.key, estimate.remainingMedianMs);
112
- }
113
- ```
114
-
115
- The estimator returns no measured ETA until a track has a known total and enough positive progress samples. A source-provided `sourceEstimatedFinishAt` remains available through the normal progress contract.
116
-
117
- ## Data boundary
118
-
119
- Snapshots may contain identifiers, phases, counters, totals, timestamps, and cost totals. Do not put credentials, environment values, input records, model responses, logs, or private content in a snapshot. The strict validator rejects unknown fields and limits the encoded snapshot to 64 KiB.
@@ -1,176 +0,0 @@
1
- # Durable job progress
2
-
3
- ## Problem
4
-
5
- Long-running remote jobs can expose state only through logs and final receipts. A monitor can confirm that a job is running, but it cannot give a reliable progress value or remaining time when the job does not publish a completed count, a total, and source timestamps.
6
-
7
- xTap Pool and OurModels need the same durable progress contract. The contract must work with their existing Hugging Face Buckets, survive worker restarts, and use the ETA estimator that Pi Workflows already uses for `pi-workflows.progress.v1` tracks.
8
-
9
- ## Requirements
10
-
11
- The implementation must:
12
-
13
- - add one storage-neutral job progress API to `@osolmaz/pi-workflows`
14
- - reuse `pi-workflows.progress.v1` for progress tracks
15
- - write one mutable snapshot for each physical job in the application's existing Bucket
16
- - let monitors discover the snapshot from immutable job labels
17
- - let Pi Workflows validate the snapshot and estimate remaining time from repeated samples
18
- - let xTap Pool report restore, review, recovery, and publication progress
19
- - let OurModels report discovery, processing, cache, and publication progress
20
- - keep receipts, content hashes, manifests, and databases authoritative
21
- - avoid secrets, post text, model output, and other private content in progress snapshots
22
- - preserve one physical enrichment job at a time during the xTap Pool change
23
- - leave the OurModels replacement schedule suspended until a paid run is separately approved
24
-
25
- ## Non-goals
26
-
27
- This work does not add:
28
-
29
- - a new remote store
30
- - a metrics database
31
- - a second ETA protocol
32
- - a Pi core change
33
- - a service or daemon
34
- - a compatibility path for an older snapshot schema
35
- - automatic authority to retry, deploy, publish, or spend money
36
-
37
- ## Public contract
38
-
39
- The npm package exports a new subpath:
40
-
41
- ```ts
42
- import {
43
- createJobProgressReporter,
44
- estimateJobProgress,
45
- validateJobProgressSnapshot,
46
- type JobProgressSnapshot,
47
- } from "@osolmaz/pi-workflows/job-progress";
48
- ```
49
-
50
- A snapshot has schema `pi-workflows.job-progress.v1` and contains:
51
-
52
- - stable application and component names
53
- - the physical job identifier
54
- - source and work-contract identifiers
55
- - a monotonic sequence number
56
- - job state and current phase
57
- - start, update, optional deadline, and optional finish timestamps
58
- - one or more keyed `pi-workflows.progress.v1` tracks
59
- - optional settled cost and active reservation facts
60
-
61
- The validator is strict. It rejects unknown fields, duplicate track keys, invalid timestamps, non-finite values, invalid progress tracks, and oversized snapshots.
62
-
63
- The reporter accepts an injected asynchronous `publish(snapshot)` callback. Pi Workflows does not import a Hugging Face client. The reporter:
64
-
65
- - keeps sequence numbers monotonic within the process
66
- - rejects regressions within one phase and epoch
67
- - coalesces frequent updates with a configurable minimum interval
68
- - flushes phase changes and terminal states immediately
69
- - bounds publication time with an abort deadline
70
- - preserves the most recent unsent snapshot after a transient publication failure
71
- - never includes arbitrary metadata or environment values
72
-
73
- A terminal snapshot is operational evidence only. The application's receipt and durable output validation still decide whether work succeeded.
74
-
75
- ## Storage and discovery
76
-
77
- Each application writes snapshots to its existing Bucket.
78
-
79
- xTap Pool uses:
80
-
81
- ```text
82
- osolmaz/xtap-pool-bucket/operations/enrichment/runs/<job-id>/progress.json
83
- ```
84
-
85
- OurModels uses:
86
-
87
- ```text
88
- osolmaz/ourmodels-data/<prefix>/operations/community-posts/runs/<job-id>/progress.json
89
- ```
90
-
91
- Each schedule supplies these immutable labels:
92
-
93
- ```text
94
- progress_schema=pi-workflows.job-progress.v1
95
- progress_bucket=<bucket>
96
- progress_prefix=<path-before-job-id>
97
- ```
98
-
99
- A monitor reads the labels, appends the physical job identifier and `progress.json`, reads the snapshot with existing local Hugging Face authentication, validates it, and publishes each track under a stable workflow progress key. The monitor does not trust an ETA string from logs. It uses source finish time when the snapshot provides one, or the existing conservative estimator after enough measured samples.
100
-
101
- ## xTap Pool tracks
102
-
103
- The enrichment worker reports these stable tracks when facts are measurable:
104
-
105
- | Key | Unit | Source |
106
- | ------------------ | ---------- | -------------------------------------------------- |
107
- | `database-restore` | bytes | downloaded database bytes and expected object size |
108
- | `registry-replay` | events | replayed registry events and discovered total |
109
- | `registry-scan` | candidates | durable scan cursor and fixed candidate total |
110
- | `queue` | records | terminal records and durable queue total |
111
- | `publication` | bytes | uploaded and verified database bytes |
112
- | `overall` | phases | completed phases and fixed phase count |
113
-
114
- Phase-only states remain valid when a total is not yet known. The worker must not invent totals.
115
-
116
- The existing three unresolved records must receive durable outcomes under the fixed full-response deadline or become exactly validated blocked records. The worker then publishes and verifies the index before its final receipt is accepted.
117
-
118
- ## OurModels tracks
119
-
120
- The community-posts worker reports these stable tracks when facts are measurable:
121
-
122
- | Key | Unit | Source |
123
- | ----------------- | ------- | ----------------------------------------------- |
124
- | `model-discovery` | models | discovered and inspected model count |
125
- | `community-posts` | models | processed models and fixed discovered total |
126
- | `cache` | records | durable cached model results and expected total |
127
- | `publication` | bytes | uploaded and verified artifact bytes |
128
- | `overall` | phases | completed phases and fixed phase count |
129
-
130
- The worker continues to use its existing receipt and manifest rules. A progress write failure must not corrupt a useful checkpoint or published result.
131
-
132
- ## Delivery sequence
133
-
134
- 1. Add and release `@osolmaz/pi-workflows/job-progress` as version `0.8.0`.
135
- 2. Add snapshot discovery guidance to the bundled monitor skill.
136
- 3. Update xTap Pool to use `0.8.0`, add measured callbacks, and add schedule labels.
137
- 4. Merge and deploy xTap Pool, then replace the old suspended schedule.
138
- 5. End the old physical job only when the replacement source is ready and the one-job rule can be preserved.
139
- 6. Start one instrumented xTap Pool recovery job under the existing restoration budget.
140
- 7. Update OurModels to use `0.8.0`, add measured callbacks, and replace old schedules with one suspended instrumented schedule.
141
- 8. Do not start a paid OurModels run until its measured cost range and ceiling receive the required approval.
142
- 9. Start a Pi monitor that reads both progress surfaces and displays current progress and ETA.
143
-
144
- ## Acceptance checks
145
-
146
- Pi Workflows must pass:
147
-
148
- ```bash
149
- npm run check
150
- npm run test:e2e
151
- npx slophammer-ts@latest dry .
152
- npx slophammer-ts@latest check . --only ts.dependency-boundaries-required
153
- ```
154
-
155
- Tests must cover strict validation, unknown fields, duplicate keys, monotonic updates, phase resets, coalescing, transient publication failure, deadline abort, terminal flush, and ETA estimation from snapshots.
156
-
157
- xTap Pool must pass its repository checks, including mutation testing. A live recovery must show a valid snapshot in the existing index Bucket, and repeated monitor samples must produce a measured ETA when a track has a known total and positive progress.
158
-
159
- OurModels must pass:
160
-
161
- ```bash
162
- npm run check
163
- npm run coverage
164
- npm run dry
165
- npm run mutate
166
- node scripts/test-bounds-engine.mjs
167
- node scripts/validate-model-data.mjs
168
- ```
169
-
170
- Its replacement schedule must remain suspended with only the approved two secrets until a paid run is authorized.
171
-
172
- ## Recovery
173
-
174
- If a progress publication fails, the worker keeps useful work and retries only the latest snapshot at the next bounded update. If the job ends before that succeeds, the receipt and durable outputs remain authoritative and the monitor marks progress stale.
175
-
176
- If a repeated deterministic worker defect appears, stop the affected job and schedule. Preserve all existing Bucket objects and return the exact failing phase, source revision, snapshot, receipt state, and last valid durable outputs.
@@ -1,24 +0,0 @@
1
- export {
2
- createJobProgressReporter,
3
- estimateJobProgress,
4
- type JobProgressReporter,
5
- } from "./reporter.js";
6
- export {
7
- isTerminalJobProgressState,
8
- MAX_JOB_PROGRESS_BYTES,
9
- MAX_JOB_PROGRESS_TRACKS,
10
- validateJobProgressSnapshot,
11
- } from "./validation.js";
12
- export {
13
- JOB_PROGRESS_SCHEMA,
14
- type JobProgressCost,
15
- type JobProgressEstimate,
16
- type JobProgressIdentity,
17
- type JobProgressPublish,
18
- type JobProgressPublishResult,
19
- type JobProgressReporterOptions,
20
- type JobProgressSnapshot,
21
- type JobProgressState,
22
- type JobProgressTrack,
23
- type JobProgressUpdate,
24
- } from "./types.js";