@on-par/factory-core 0.9.27-next.1 → 0.9.27-next.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.d.ts +4 -27
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -16
- package/dist/index.js.map +1 -1
- package/package.json +5 -5
- package/dist/hosted/control-plane.d.ts +0 -40
- package/dist/hosted/control-plane.d.ts.map +0 -1
- package/dist/hosted/control-plane.js +0 -198
- package/dist/hosted/control-plane.js.map +0 -1
- package/dist/hosted/queue-client.d.ts +0 -51
- package/dist/hosted/queue-client.d.ts.map +0 -1
- package/dist/hosted/queue-client.js +0 -92
- package/dist/hosted/queue-client.js.map +0 -1
- package/dist/hosted/runner-client.d.ts +0 -78
- package/dist/hosted/runner-client.d.ts.map +0 -1
- package/dist/hosted/runner-client.js +0 -82
- package/dist/hosted/runner-client.js.map +0 -1
- package/dist/hosted/runner.d.ts +0 -74
- package/dist/hosted/runner.d.ts.map +0 -1
- package/dist/hosted/runner.js +0 -174
- package/dist/hosted/runner.js.map +0 -1
- package/dist/hosted/smoke.d.ts +0 -46
- package/dist/hosted/smoke.d.ts.map +0 -1
- package/dist/hosted/smoke.js +0 -56
- package/dist/hosted/smoke.js.map +0 -1
- package/dist/hosted/store-resolve.d.ts +0 -16
- package/dist/hosted/store-resolve.d.ts.map +0 -1
- package/dist/hosted/store-resolve.js +0 -13
- package/dist/hosted/store-resolve.js.map +0 -1
- package/dist/hosted/store-sqlite.d.ts +0 -10
- package/dist/hosted/store-sqlite.d.ts.map +0 -1
- package/dist/hosted/store-sqlite.js +0 -67
- package/dist/hosted/store-sqlite.js.map +0 -1
- package/dist/hosted/summary.d.ts +0 -26
- package/dist/hosted/summary.d.ts.map +0 -1
- package/dist/hosted/summary.js +0 -27
- package/dist/hosted/summary.js.map +0 -1
- package/dist/hosted/watchdog.d.ts +0 -38
- package/dist/hosted/watchdog.d.ts.map +0 -1
- package/dist/hosted/watchdog.js +0 -56
- package/dist/hosted/watchdog.js.map +0 -1
|
@@ -1,67 +0,0 @@
|
|
|
1
|
-
// src/hosted/store-sqlite.ts — node:sqlite-backed persistence for the hosted
|
|
2
|
-
// job store (#939). Durability across process restarts, no new dependency:
|
|
3
|
-
// node:sqlite (DatabaseSync) ships built into Node >=22.5 / core's >=24
|
|
4
|
-
// engines requirement. All idempotency/lease invariant logic lives in
|
|
5
|
-
// createHostedJobStoreOverPersistence (store.ts); this file only implements
|
|
6
|
-
// the HostedStorePersistence port over two JSON-blob tables.
|
|
7
|
-
import { DatabaseSync } from 'node:sqlite';
|
|
8
|
-
import { createHostedJobStoreOverPersistence, } from './store.js';
|
|
9
|
-
function bootstrapSchema(db) {
|
|
10
|
-
const row = db.prepare('PRAGMA user_version').get();
|
|
11
|
-
const version = row?.user_version ?? 0;
|
|
12
|
-
if (version === 0) {
|
|
13
|
-
db.exec(`
|
|
14
|
-
CREATE TABLE IF NOT EXISTS jobs (id TEXT PRIMARY KEY, data TEXT NOT NULL);
|
|
15
|
-
CREATE TABLE IF NOT EXISTS runners (id TEXT PRIMARY KEY, data TEXT NOT NULL);
|
|
16
|
-
PRAGMA user_version = 1;
|
|
17
|
-
`);
|
|
18
|
-
}
|
|
19
|
-
}
|
|
20
|
-
function createSqlitePersistence(db) {
|
|
21
|
-
const hasJobStmt = db.prepare('SELECT 1 FROM jobs WHERE id = ?');
|
|
22
|
-
const getJobStmt = db.prepare('SELECT data FROM jobs WHERE id = ?');
|
|
23
|
-
const putJobStmt = db.prepare('INSERT INTO jobs (id, data) VALUES (?, ?) ON CONFLICT(id) DO UPDATE SET data = excluded.data');
|
|
24
|
-
const allJobsStmt = db.prepare('SELECT data FROM jobs');
|
|
25
|
-
const getRunnerStmt = db.prepare('SELECT data FROM runners WHERE id = ?');
|
|
26
|
-
const putRunnerStmt = db.prepare('INSERT INTO runners (id, data) VALUES (?, ?) ON CONFLICT(id) DO UPDATE SET data = excluded.data');
|
|
27
|
-
const allRunnersStmt = db.prepare('SELECT data FROM runners');
|
|
28
|
-
return {
|
|
29
|
-
hasJob(jobId) {
|
|
30
|
-
return hasJobStmt.get(jobId) !== undefined;
|
|
31
|
-
},
|
|
32
|
-
getJob(jobId) {
|
|
33
|
-
const row = getJobStmt.get(jobId);
|
|
34
|
-
return row ? JSON.parse(row.data) : undefined;
|
|
35
|
-
},
|
|
36
|
-
putJob(job) {
|
|
37
|
-
putJobStmt.run(job.request.jobId, JSON.stringify(job));
|
|
38
|
-
},
|
|
39
|
-
jobs() {
|
|
40
|
-
const rows = allJobsStmt.all();
|
|
41
|
-
return rows.map((row) => JSON.parse(row.data));
|
|
42
|
-
},
|
|
43
|
-
getRunner(runnerId) {
|
|
44
|
-
const row = getRunnerStmt.get(runnerId);
|
|
45
|
-
return row ? JSON.parse(row.data) : undefined;
|
|
46
|
-
},
|
|
47
|
-
putRunner(runner) {
|
|
48
|
-
putRunnerStmt.run(runner.runnerId, JSON.stringify(runner));
|
|
49
|
-
},
|
|
50
|
-
runners() {
|
|
51
|
-
const rows = allRunnersStmt.all();
|
|
52
|
-
return rows.map((row) => JSON.parse(row.data));
|
|
53
|
-
},
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
-
export function createSqliteHostedJobStore(options) {
|
|
57
|
-
const { databasePath, ...storeOptions } = options;
|
|
58
|
-
const db = new DatabaseSync(databasePath ?? ':memory:');
|
|
59
|
-
bootstrapSchema(db);
|
|
60
|
-
const persistence = createSqlitePersistence(db);
|
|
61
|
-
const store = createHostedJobStoreOverPersistence(persistence, storeOptions);
|
|
62
|
-
return {
|
|
63
|
-
...store,
|
|
64
|
-
close: () => db.close(),
|
|
65
|
-
};
|
|
66
|
-
}
|
|
67
|
-
//# sourceMappingURL=store-sqlite.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"store-sqlite.js","sourceRoot":"","sources":["../../src/hosted/store-sqlite.ts"],"names":[],"mappings":"AAAA,6EAA6E;AAC7E,2EAA2E;AAC3E,wEAAwE;AACxE,sEAAsE;AACtE,4EAA4E;AAC5E,6DAA6D;AAC7D,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAE3C,OAAO,EACL,mCAAmC,GAMpC,MAAM,YAAY,CAAC;AASpB,SAAS,eAAe,CAAC,EAAqC;IAC5D,MAAM,GAAG,GAAG,EAAE,CAAC,OAAO,CAAC,qBAAqB,CAAC,CAAC,GAAG,EAA0C,CAAC;IAC5F,MAAM,OAAO,GAAG,GAAG,EAAE,YAAY,IAAI,CAAC,CAAC;IACvC,IAAI,OAAO,KAAK,CAAC,EAAE,CAAC;QAClB,EAAE,CAAC,IAAI,CAAC;;;;KAIP,CAAC,CAAC;IACL,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,EAAqC;IACpE,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,iCAAiC,CAAC,CAAC;IACjE,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAAC,oCAAoC,CAAC,CAAC;IACpE,MAAM,UAAU,GAAG,EAAE,CAAC,OAAO,CAC3B,8FAA8F,CAC/F,CAAC;IACF,MAAM,WAAW,GAAG,EAAE,CAAC,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAExD,MAAM,aAAa,GAAG,EAAE,CAAC,OAAO,CAAC,uCAAuC,CAAC,CAAC;IAC1E,MAAM,aAAa,GAAG,EAAE,CAAC,OAAO,CAC9B,iGAAiG,CAClG,CAAC;IACF,MAAM,cAAc,GAAG,EAAE,CAAC,OAAO,CAAC,0BAA0B,CAAC,CAAC;IAE9D,OAAO;QACL,MAAM,CAAC,KAAK;YACV,OAAO,UAAU,CAAC,GAAG,CAAC,KAAK,CAAC,KAAK,SAAS,CAAC;QAC7C,CAAC;QACD,MAAM,CAAC,KAAK;YACV,MAAM,GAAG,GAAG,UAAU,CAAC,GAAG,CAAC,KAAK,CAAiC,CAAC;YAClE,OAAO,GAAG,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAqB,CAAC,CAAC,CAAC,SAAS,CAAC;QACrE,CAAC;QACD,MAAM,CAAC,GAAG;YACR,UAAU,CAAC,GAAG,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,CAAC,CAAC;QACzD,CAAC;QACD,IAAI;YACF,MAAM,IAAI,GAAG,WAAW,CAAC,GAAG,EAAwB,CAAC;YACrD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAoB,CAAC,CAAC;QACpE,CAAC;QACD,SAAS,CAAC,QAAQ;YAChB,MAAM,GAAG,GAAG,aAAa,CAAC,GAAG,CAAC,QAAQ,CAAiC,CAAC;YACxE,OAAO,GAAG,CAAC,CAAC,CAAE,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAkB,CAAC,CAAC,CAAC,SAAS,CAAC;QAClE,CAAC;QACD,SAAS,CAAC,MAAM;YACd,aAAa,CAAC,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC;QAC7D,CAAC;QACD,OAAO;YACL,MAAM,IAAI,GAAG,cAAc,CAAC,GAAG,EAAwB,CAAC;YACxD,OAAO,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,IAAI,CAAiB,CAAC,CAAC;QACjE,CAAC;KACF,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,0BAA0B,CAAC,OAAoC;IAC7E,MAAM,EAAE,YAAY,EAAE,GAAG,YAAY,EAAE,GAAG,OAAO,CAAC;IAClD,MAAM,EAAE,GAAG,IAAI,YAAY,CAAC,YAAY,IAAI,UAAU,CAAC,CAAC;IACxD,eAAe,CAAC,EAAE,CAAC,CAAC;IACpB,MAAM,WAAW,GAAG,uBAAuB,CAAC,EAAE,CAAC,CAAC;IAChD,MAAM,KAAK,GAAG,mCAAmC,CAAC,WAAW,EAAE,YAAY,CAAC,CAAC;IAC7E,OAAO;QACL,GAAG,KAAK;QACR,KAAK,EAAE,GAAG,EAAE,CAAC,EAAE,CAAC,KAAK,EAAE;KACxB,CAAC;AACJ,CAAC"}
|
package/dist/hosted/summary.d.ts
DELETED
|
@@ -1,26 +0,0 @@
|
|
|
1
|
-
import type { HostedArtifactRef, HostedJobEvent, HostedJobOutcome, HostedJobPhase, HostedJobStatus } from '@on-par/contracts';
|
|
2
|
-
import type { StoredHostedJob } from './store.js';
|
|
3
|
-
export interface HostedJobSummary {
|
|
4
|
-
jobId: string;
|
|
5
|
-
repoSlug: string;
|
|
6
|
-
status: HostedJobStatus;
|
|
7
|
-
/** runnerId currently holding the active lease, else null. */
|
|
8
|
-
leasedBy: string | null;
|
|
9
|
-
outcome: HostedJobOutcome | null;
|
|
10
|
-
summary: string | null;
|
|
11
|
-
exitCode: number | null;
|
|
12
|
-
failurePhase: HostedJobPhase | null;
|
|
13
|
-
eventCount: number;
|
|
14
|
-
lastEvent: HostedJobEvent | null;
|
|
15
|
-
artifacts: HostedArtifactRef[];
|
|
16
|
-
artifactCount: number;
|
|
17
|
-
logsTail: string | null;
|
|
18
|
-
/** Message of the last 'cleaned' event, else null. */
|
|
19
|
-
cleanupProof: string | null;
|
|
20
|
-
updatedAt: string;
|
|
21
|
-
}
|
|
22
|
-
/** Maps a stored job to an operator-facing summary. Exposes no secrets: only
|
|
23
|
-
* metadata already present on the retained record (result/events). */
|
|
24
|
-
export declare function summarizeHostedJob(job: StoredHostedJob): HostedJobSummary;
|
|
25
|
-
export declare function summarizeHostedJobs(jobs: StoredHostedJob[]): HostedJobSummary[];
|
|
26
|
-
//# sourceMappingURL=summary.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"summary.d.ts","sourceRoot":"","sources":["../../src/hosted/summary.ts"],"names":[],"mappings":"AAIA,OAAO,KAAK,EACV,iBAAiB,EACjB,cAAc,EACd,gBAAgB,EAChB,cAAc,EACd,eAAe,EAChB,MAAM,mBAAmB,CAAC;AAE3B,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,YAAY,CAAC;AAElD,MAAM,WAAW,gBAAgB;IAC/B,KAAK,EAAE,MAAM,CAAC;IACd,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,eAAe,CAAC;IACxB,8DAA8D;IAC9D,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,OAAO,EAAE,gBAAgB,GAAG,IAAI,CAAC;IACjC,OAAO,EAAE,MAAM,GAAG,IAAI,CAAC;IACvB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,YAAY,EAAE,cAAc,GAAG,IAAI,CAAC;IACpC,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,cAAc,GAAG,IAAI,CAAC;IACjC,SAAS,EAAE,iBAAiB,EAAE,CAAC;IAC/B,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,sDAAsD;IACtD,YAAY,EAAE,MAAM,GAAG,IAAI,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;CACnB;AAED;sEACsE;AACtE,wBAAgB,kBAAkB,CAAC,GAAG,EAAE,eAAe,GAAG,gBAAgB,CAoBzE;AAED,wBAAgB,mBAAmB,CAAC,IAAI,EAAE,eAAe,EAAE,GAAG,gBAAgB,EAAE,CAE/E"}
|
package/dist/hosted/summary.js
DELETED
|
@@ -1,27 +0,0 @@
|
|
|
1
|
-
/** Maps a stored job to an operator-facing summary. Exposes no secrets: only
|
|
2
|
-
* metadata already present on the retained record (result/events). */
|
|
3
|
-
export function summarizeHostedJob(job) {
|
|
4
|
-
const artifacts = job.result?.artifacts ?? [];
|
|
5
|
-
const cleaned = [...job.events].reverse().find((event) => event.type === 'cleaned');
|
|
6
|
-
return {
|
|
7
|
-
jobId: job.request.jobId,
|
|
8
|
-
repoSlug: job.request.repoSlug,
|
|
9
|
-
status: job.request.status,
|
|
10
|
-
leasedBy: job.lease?.runnerId ?? null,
|
|
11
|
-
outcome: job.result?.outcome ?? null,
|
|
12
|
-
summary: job.result?.summary ?? null,
|
|
13
|
-
exitCode: job.result?.exitCode ?? null,
|
|
14
|
-
failurePhase: job.result?.failurePhase ?? null,
|
|
15
|
-
eventCount: job.events.length,
|
|
16
|
-
lastEvent: job.events.at(-1) ?? null,
|
|
17
|
-
artifacts,
|
|
18
|
-
artifactCount: artifacts.length,
|
|
19
|
-
logsTail: job.result?.logsTail ?? null,
|
|
20
|
-
cleanupProof: cleaned?.message ?? null,
|
|
21
|
-
updatedAt: job.updatedAt,
|
|
22
|
-
};
|
|
23
|
-
}
|
|
24
|
-
export function summarizeHostedJobs(jobs) {
|
|
25
|
-
return jobs.map(summarizeHostedJob);
|
|
26
|
-
}
|
|
27
|
-
//# sourceMappingURL=summary.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"summary.js","sourceRoot":"","sources":["../../src/hosted/summary.ts"],"names":[],"mappings":"AAkCA;sEACsE;AACtE,MAAM,UAAU,kBAAkB,CAAC,GAAoB;IACrD,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,EAAE,SAAS,IAAI,EAAE,CAAC;IAC9C,MAAM,OAAO,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,SAAS,CAAC,CAAC;IACpF,OAAO;QACL,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK;QACxB,QAAQ,EAAE,GAAG,CAAC,OAAO,CAAC,QAAQ;QAC9B,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM;QAC1B,QAAQ,EAAE,GAAG,CAAC,KAAK,EAAE,QAAQ,IAAI,IAAI;QACrC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,IAAI,IAAI;QACpC,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,OAAO,IAAI,IAAI;QACpC,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,IAAI,IAAI;QACtC,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,YAAY,IAAI,IAAI;QAC9C,UAAU,EAAE,GAAG,CAAC,MAAM,CAAC,MAAM;QAC7B,SAAS,EAAE,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,IAAI;QACpC,SAAS;QACT,aAAa,EAAE,SAAS,CAAC,MAAM;QAC/B,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,QAAQ,IAAI,IAAI;QACtC,YAAY,EAAE,OAAO,EAAE,OAAO,IAAI,IAAI;QACtC,SAAS,EAAE,GAAG,CAAC,SAAS;KACzB,CAAC;AACJ,CAAC;AAED,MAAM,UAAU,mBAAmB,CAAC,IAAuB;IACzD,OAAO,IAAI,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;AACtC,CAAC"}
|
|
@@ -1,38 +0,0 @@
|
|
|
1
|
-
import type { HostedJobEvent, HostedJobStatus } from '@on-par/contracts';
|
|
2
|
-
import type { HostedJobStore } from './store.js';
|
|
3
|
-
export interface WatchdogPolicy {
|
|
4
|
-
/** ms clock, injected for determinism. */
|
|
5
|
-
now: () => number;
|
|
6
|
-
/** A lease-holding runner is "dead" if its lastHeartbeatAt is older than this. */
|
|
7
|
-
runnerHeartbeatTimeoutMs: number;
|
|
8
|
-
/** Hard cap on a single lease's runtime. */
|
|
9
|
-
maxJobRuntimeMs: number;
|
|
10
|
-
/** Max 'leased' events before a dead-runner job escalates instead of relaunching. */
|
|
11
|
-
maxRelaunches: number;
|
|
12
|
-
/** Events included in an escalation's recentEvents tail. Default 5. */
|
|
13
|
-
recentEventLimit?: number;
|
|
14
|
-
}
|
|
15
|
-
export interface WatchdogEscalation {
|
|
16
|
-
jobId: string;
|
|
17
|
-
status: HostedJobStatus;
|
|
18
|
-
reason: string;
|
|
19
|
-
recentEvents: HostedJobEvent[];
|
|
20
|
-
manualIntervention: string;
|
|
21
|
-
}
|
|
22
|
-
export interface WatchdogReport {
|
|
23
|
-
/** ISO-8601 of policy.now(). */
|
|
24
|
-
sweptAt: string;
|
|
25
|
-
/** jobIds returned to retryable via lease expiry. */
|
|
26
|
-
reclaimedExpired: string[];
|
|
27
|
-
/** jobIds force-reclaimed due to a dead runner. */
|
|
28
|
-
relaunchedDeadRunners: string[];
|
|
29
|
-
/** jobIds failed for exceeding maxJobRuntimeMs. */
|
|
30
|
-
timedOut: string[];
|
|
31
|
-
/** Stuck runs the watchdog cannot safely fix. */
|
|
32
|
-
escalations: WatchdogEscalation[];
|
|
33
|
-
}
|
|
34
|
-
/** Runs one reliability sweep over every non-terminal job in the store,
|
|
35
|
-
* classifying each into reclaimed-expired / relaunched-dead-runner /
|
|
36
|
-
* timed-out / escalation. */
|
|
37
|
-
export declare function runWatchdogSweep(store: HostedJobStore, policy: WatchdogPolicy): WatchdogReport;
|
|
38
|
-
//# sourceMappingURL=watchdog.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"watchdog.d.ts","sourceRoot":"","sources":["../../src/hosted/watchdog.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAE,cAAc,EAAE,eAAe,EAAE,MAAM,mBAAmB,CAAC;AAEzE,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,YAAY,CAAC;AAEjD,MAAM,WAAW,cAAc;IAC7B,0CAA0C;IAC1C,GAAG,EAAE,MAAM,MAAM,CAAC;IAClB,kFAAkF;IAClF,wBAAwB,EAAE,MAAM,CAAC;IACjC,4CAA4C;IAC5C,eAAe,EAAE,MAAM,CAAC;IACxB,qFAAqF;IACrF,aAAa,EAAE,MAAM,CAAC;IACtB,uEAAuE;IACvE,gBAAgB,CAAC,EAAE,MAAM,CAAC;CAC3B;AAED,MAAM,WAAW,kBAAkB;IACjC,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,eAAe,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,YAAY,EAAE,cAAc,EAAE,CAAC;IAC/B,kBAAkB,EAAE,MAAM,CAAC;CAC5B;AAED,MAAM,WAAW,cAAc;IAC7B,gCAAgC;IAChC,OAAO,EAAE,MAAM,CAAC;IAChB,qDAAqD;IACrD,gBAAgB,EAAE,MAAM,EAAE,CAAC;IAC3B,mDAAmD;IACnD,qBAAqB,EAAE,MAAM,EAAE,CAAC;IAChC,mDAAmD;IACnD,QAAQ,EAAE,MAAM,EAAE,CAAC;IACnB,iDAAiD;IACjD,WAAW,EAAE,kBAAkB,EAAE,CAAC;CACnC;AAWD;;6BAE6B;AAC7B,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,cAAc,EAAE,MAAM,EAAE,cAAc,GAAG,cAAc,CAgD9F"}
|
package/dist/hosted/watchdog.js
DELETED
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
function leasedAtMs(job) {
|
|
2
|
-
const lastLeased = [...job.events].reverse().find((event) => event.type === 'leased');
|
|
3
|
-
return lastLeased ? Date.parse(lastLeased.ts) : Date.parse(job.updatedAt);
|
|
4
|
-
}
|
|
5
|
-
function leaseCount(job) {
|
|
6
|
-
return job.events.filter((event) => event.type === 'leased').length;
|
|
7
|
-
}
|
|
8
|
-
/** Runs one reliability sweep over every non-terminal job in the store,
|
|
9
|
-
* classifying each into reclaimed-expired / relaunched-dead-runner /
|
|
10
|
-
* timed-out / escalation. */
|
|
11
|
-
export function runWatchdogSweep(store, policy) {
|
|
12
|
-
const reclaimedExpired = store.reclaimExpired().map((job) => job.request.jobId);
|
|
13
|
-
const relaunchedDeadRunners = [];
|
|
14
|
-
const timedOut = [];
|
|
15
|
-
const escalations = [];
|
|
16
|
-
const recentEventLimit = policy.recentEventLimit ?? 5;
|
|
17
|
-
for (const job of store.list()) {
|
|
18
|
-
if (job.lease === null) {
|
|
19
|
-
continue;
|
|
20
|
-
}
|
|
21
|
-
const runtimeMs = policy.now() - leasedAtMs(job);
|
|
22
|
-
const runner = store.getRunner(job.lease.runnerId);
|
|
23
|
-
const runnerDead = !runner || policy.now() - Date.parse(runner.lastHeartbeatAt) > policy.runnerHeartbeatTimeoutMs;
|
|
24
|
-
if (runtimeMs > policy.maxJobRuntimeMs) {
|
|
25
|
-
store.fail(job.request.jobId, job.lease.leaseId, `watchdog: exceeded max runtime ${policy.maxJobRuntimeMs}ms`, {
|
|
26
|
-
failurePhase: 'run',
|
|
27
|
-
});
|
|
28
|
-
timedOut.push(job.request.jobId);
|
|
29
|
-
continue;
|
|
30
|
-
}
|
|
31
|
-
if (runnerDead) {
|
|
32
|
-
const count = leaseCount(job);
|
|
33
|
-
if (count < policy.maxRelaunches) {
|
|
34
|
-
store.reclaimJob(job.request.jobId, 'runner heartbeat stale');
|
|
35
|
-
relaunchedDeadRunners.push(job.request.jobId);
|
|
36
|
-
}
|
|
37
|
-
else {
|
|
38
|
-
escalations.push({
|
|
39
|
-
jobId: job.request.jobId,
|
|
40
|
-
status: job.request.status,
|
|
41
|
-
reason: 'dead runner and relaunch budget exhausted',
|
|
42
|
-
recentEvents: job.events.slice(-recentEventLimit),
|
|
43
|
-
manualIntervention: `inspect .factory events for ${job.request.jobId}; cancel or manually re-lease`,
|
|
44
|
-
});
|
|
45
|
-
}
|
|
46
|
-
}
|
|
47
|
-
}
|
|
48
|
-
return {
|
|
49
|
-
sweptAt: new Date(policy.now()).toISOString(),
|
|
50
|
-
reclaimedExpired,
|
|
51
|
-
relaunchedDeadRunners,
|
|
52
|
-
timedOut,
|
|
53
|
-
escalations,
|
|
54
|
-
};
|
|
55
|
-
}
|
|
56
|
-
//# sourceMappingURL=watchdog.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"watchdog.js","sourceRoot":"","sources":["../../src/hosted/watchdog.ts"],"names":[],"mappings":"AA4CA,SAAS,UAAU,CAAC,GAA+C;IACjE,MAAM,UAAU,GAAG,CAAC,GAAG,GAAG,CAAC,MAAM,CAAC,CAAC,OAAO,EAAE,CAAC,IAAI,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC;IACtF,OAAO,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,UAAU,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;AAC5E,CAAC;AAED,SAAS,UAAU,CAAC,GAA+C;IACjE,OAAO,GAAG,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,KAAK,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,MAAM,CAAC;AACtE,CAAC;AAED;;6BAE6B;AAC7B,MAAM,UAAU,gBAAgB,CAAC,KAAqB,EAAE,MAAsB;IAC5E,MAAM,gBAAgB,GAAG,KAAK,CAAC,cAAc,EAAE,CAAC,GAAG,CAAC,CAAC,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;IAEhF,MAAM,qBAAqB,GAAa,EAAE,CAAC;IAC3C,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,WAAW,GAAyB,EAAE,CAAC;IAC7C,MAAM,gBAAgB,GAAG,MAAM,CAAC,gBAAgB,IAAI,CAAC,CAAC;IAEtD,KAAK,MAAM,GAAG,IAAI,KAAK,CAAC,IAAI,EAAE,EAAE,CAAC;QAC/B,IAAI,GAAG,CAAC,KAAK,KAAK,IAAI,EAAE,CAAC;YACvB,SAAS;QACX,CAAC;QACD,MAAM,SAAS,GAAG,MAAM,CAAC,GAAG,EAAE,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;QACjD,MAAM,MAAM,GAAG,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACnD,MAAM,UAAU,GAAG,CAAC,MAAM,IAAI,MAAM,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC,KAAK,CAAC,MAAM,CAAC,eAAe,CAAC,GAAG,MAAM,CAAC,wBAAwB,CAAC;QAElH,IAAI,SAAS,GAAG,MAAM,CAAC,eAAe,EAAE,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,GAAG,CAAC,KAAK,CAAC,OAAO,EAAE,kCAAkC,MAAM,CAAC,eAAe,IAAI,EAAE;gBAC7G,YAAY,EAAE,KAAK;aACpB,CAAC,CAAC;YACH,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YACjC,SAAS;QACX,CAAC;QAED,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,KAAK,GAAG,MAAM,CAAC,aAAa,EAAE,CAAC;gBACjC,KAAK,CAAC,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,EAAE,wBAAwB,CAAC,CAAC;gBAC9D,qBAAqB,CAAC,IAAI,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;YAChD,CAAC;iBAAM,CAAC;gBACN,WAAW,CAAC,IAAI,CAAC;oBACf,KAAK,EAAE,GAAG,CAAC,OAAO,CAAC,KAAK;oBACxB,MAAM,EAAE,GAAG,CAAC,OAAO,CAAC,MAAM;oBAC1B,MAAM,EAAE,2CAA2C;oBACnD,YAAY,EAAE,GAAG,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,gBAAgB,CAAC;oBACjD,kBAAkB,EAAE,+BAA+B,GAAG,CAAC,OAAO,CAAC,KAAK,+BAA+B;iBACpG,CAAC,CAAC;YACL,CAAC;QACH,CAAC;IACH,CAAC;IAED,OAAO;QACL,OAAO,EAAE,IAAI,IAAI,CAAC,MAAM,CAAC,GAAG,EAAE,CAAC,CAAC,WAAW,EAAE;QAC7C,gBAAgB;QAChB,qBAAqB;QACrB,QAAQ;QACR,WAAW;KACZ,CAAC;AACJ,CAAC"}
|