@holon-run/agentinbox 1.0.0-beta.3 → 1.0.0-beta.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/src/adapters.js +49 -1
- package/dist/src/cli.js +91 -0
- package/dist/src/http.js +53 -3
- package/dist/src/service.js +301 -16
- package/dist/src/source_resolution.js +10 -0
- package/dist/src/source_schema.js +2 -0
- package/dist/src/sources/github.js +218 -6
- package/dist/src/sources/github_ci.js +48 -23
- package/dist/src/sources/remote.js +183 -1
- package/dist/src/sources/remote_modules.js +22 -3
- package/dist/src/store.js +8 -4
- package/dist/src/terminal.js +4 -3
- package/drizzle/migrations/0001_activation_entry_boundary.sql +11 -0
- package/drizzle/schema.ts +2 -0
- package/package.json +2 -2
|
@@ -6,9 +6,12 @@ exports.invokeGithubDeliveryOperation = invokeGithubDeliveryOperation;
|
|
|
6
6
|
exports.normalizeGithubRepoEvent = normalizeGithubRepoEvent;
|
|
7
7
|
exports.hydrateGithubRepoEventIfNeeded = hydrateGithubRepoEventIfNeeded;
|
|
8
8
|
exports.canonicalGithubPullRequestRef = canonicalGithubPullRequestRef;
|
|
9
|
+
exports.canonicalGithubIssueRef = canonicalGithubIssueRef;
|
|
9
10
|
exports.deriveGithubTrackedResource = deriveGithubTrackedResource;
|
|
10
11
|
exports.githubSubscriptionShortcutSpec = githubSubscriptionShortcutSpec;
|
|
12
|
+
exports.githubFollowTemplateSpec = githubFollowTemplateSpec;
|
|
11
13
|
exports.expandGithubSubscriptionShortcut = expandGithubSubscriptionShortcut;
|
|
14
|
+
exports.expandGithubFollowTemplate = expandGithubFollowTemplate;
|
|
12
15
|
exports.projectGithubLifecycleSignal = projectGithubLifecycleSignal;
|
|
13
16
|
exports.parseGithubSourceConfig = parseGithubSourceConfig;
|
|
14
17
|
const uxc_daemon_client_1 = require("@holon-run/uxc-daemon-client");
|
|
@@ -88,6 +91,24 @@ class GithubUxcClient {
|
|
|
88
91
|
options: { auth: input.auth },
|
|
89
92
|
});
|
|
90
93
|
}
|
|
94
|
+
async getPullRequestDetails(input) {
|
|
95
|
+
const response = await this.client.call({
|
|
96
|
+
endpoint: exports.GITHUB_ENDPOINT,
|
|
97
|
+
operation: "get:/repos/{owner}/{repo}/pulls/{pull_number}",
|
|
98
|
+
payload: {
|
|
99
|
+
owner: input.owner,
|
|
100
|
+
repo: input.repo,
|
|
101
|
+
pull_number: input.pullNumber,
|
|
102
|
+
},
|
|
103
|
+
options: { auth: input.auth },
|
|
104
|
+
});
|
|
105
|
+
const data = asRecord(response.data);
|
|
106
|
+
const head = asRecord(data.head);
|
|
107
|
+
return {
|
|
108
|
+
headRefName: asString(head.ref),
|
|
109
|
+
headRepositoryFullName: asString(asRecord(head.repo).full_name),
|
|
110
|
+
};
|
|
111
|
+
}
|
|
91
112
|
}
|
|
92
113
|
exports.GithubUxcClient = GithubUxcClient;
|
|
93
114
|
class GithubDeliveryAdapter {
|
|
@@ -336,6 +357,10 @@ function canonicalGithubPullRequestRef(source, number) {
|
|
|
336
357
|
const config = parseGithubSourceConfig(source);
|
|
337
358
|
return `repo:${config.owner}/${config.repo}:pr:${number}`;
|
|
338
359
|
}
|
|
360
|
+
function canonicalGithubIssueRef(source, number) {
|
|
361
|
+
const config = parseGithubSourceConfig(source);
|
|
362
|
+
return `repo:${config.owner}/${config.repo}:issue:${number}`;
|
|
363
|
+
}
|
|
339
364
|
function deriveGithubTrackedResource(filter, source) {
|
|
340
365
|
const metadata = asRecord(filter.metadata);
|
|
341
366
|
const payload = asRecord(filter.payload);
|
|
@@ -356,7 +381,43 @@ function githubSubscriptionShortcutSpec() {
|
|
|
356
381
|
],
|
|
357
382
|
}];
|
|
358
383
|
}
|
|
359
|
-
function
|
|
384
|
+
function githubFollowTemplateSpec() {
|
|
385
|
+
const repoArgs = [
|
|
386
|
+
{ name: "owner", type: "string", required: true, description: "GitHub repository owner." },
|
|
387
|
+
{ name: "repo", type: "string", required: true, description: "GitHub repository name." },
|
|
388
|
+
];
|
|
389
|
+
return [
|
|
390
|
+
{
|
|
391
|
+
templateId: "github.repo",
|
|
392
|
+
providerOrKind: "github",
|
|
393
|
+
label: "GitHub Repository",
|
|
394
|
+
description: "Follow a repository event stream.",
|
|
395
|
+
argsSchema: repoArgs,
|
|
396
|
+
},
|
|
397
|
+
{
|
|
398
|
+
templateId: "github.pr",
|
|
399
|
+
providerOrKind: "github",
|
|
400
|
+
label: "GitHub Pull Request",
|
|
401
|
+
description: "Follow one pull request and optionally its CI.",
|
|
402
|
+
argsSchema: [
|
|
403
|
+
...repoArgs,
|
|
404
|
+
{ name: "number", type: "number", required: true, description: "Pull request number." },
|
|
405
|
+
{ name: "withCi", type: "boolean", required: false, description: "Also follow the repository CI stream for that pull request." },
|
|
406
|
+
],
|
|
407
|
+
},
|
|
408
|
+
{
|
|
409
|
+
templateId: "github.issue",
|
|
410
|
+
providerOrKind: "github",
|
|
411
|
+
label: "GitHub Issue",
|
|
412
|
+
description: "Follow one issue without automatic terminal cleanup.",
|
|
413
|
+
argsSchema: [
|
|
414
|
+
...repoArgs,
|
|
415
|
+
{ name: "number", type: "number", required: true, description: "Issue number." },
|
|
416
|
+
],
|
|
417
|
+
},
|
|
418
|
+
];
|
|
419
|
+
}
|
|
420
|
+
async function expandGithubSubscriptionShortcut(input, client = new GithubUxcClient()) {
|
|
360
421
|
if (input.name !== "pr") {
|
|
361
422
|
return null;
|
|
362
423
|
}
|
|
@@ -366,6 +427,9 @@ function expandGithubSubscriptionShortcut(input) {
|
|
|
366
427
|
}
|
|
367
428
|
const trackedResourceRef = canonicalGithubPullRequestRef(input.source, number);
|
|
368
429
|
const withCi = input.args?.withCi === true;
|
|
430
|
+
const ciFilter = withCi
|
|
431
|
+
? await buildGithubPrCiFilter(input.source, number, client)
|
|
432
|
+
: null;
|
|
369
433
|
return {
|
|
370
434
|
members: [
|
|
371
435
|
{
|
|
@@ -382,11 +446,7 @@ function expandGithubSubscriptionShortcut(input) {
|
|
|
382
446
|
...(withCi
|
|
383
447
|
? [{
|
|
384
448
|
streamKind: "ci_runs",
|
|
385
|
-
filter:
|
|
386
|
-
metadata: {
|
|
387
|
-
pullRequestNumbers: [number],
|
|
388
|
-
},
|
|
389
|
-
},
|
|
449
|
+
filter: ciFilter ?? undefined,
|
|
390
450
|
trackedResourceRef,
|
|
391
451
|
cleanupPolicy: { mode: "on_terminal" },
|
|
392
452
|
}]
|
|
@@ -394,6 +454,152 @@ function expandGithubSubscriptionShortcut(input) {
|
|
|
394
454
|
],
|
|
395
455
|
};
|
|
396
456
|
}
|
|
457
|
+
async function expandGithubFollowTemplate(input, client = new GithubUxcClient()) {
|
|
458
|
+
const config = parseGithubSourceConfig(input.source);
|
|
459
|
+
if (input.template === "repo") {
|
|
460
|
+
return {
|
|
461
|
+
templateId: "github.repo",
|
|
462
|
+
sources: [
|
|
463
|
+
{
|
|
464
|
+
logicalName: "repo_events",
|
|
465
|
+
sourceType: "github_repo",
|
|
466
|
+
sourceKey: `${config.owner}/${config.repo}`,
|
|
467
|
+
configRef: input.source.configRef ?? null,
|
|
468
|
+
config: githubFollowSourceConfig(input.source),
|
|
469
|
+
},
|
|
470
|
+
],
|
|
471
|
+
subscriptions: [
|
|
472
|
+
{
|
|
473
|
+
sourceLogicalName: "repo_events",
|
|
474
|
+
filter: {},
|
|
475
|
+
cleanupPolicy: { mode: "manual" },
|
|
476
|
+
},
|
|
477
|
+
],
|
|
478
|
+
};
|
|
479
|
+
}
|
|
480
|
+
const number = typeof input.args?.number === "number" ? input.args.number : null;
|
|
481
|
+
if (input.template !== "pr" && input.template !== "issue") {
|
|
482
|
+
return null;
|
|
483
|
+
}
|
|
484
|
+
if (!number || !Number.isInteger(number) || number <= 0) {
|
|
485
|
+
throw new Error(`follow template github.${input.template} requires a positive integer number`);
|
|
486
|
+
}
|
|
487
|
+
if (input.template === "pr") {
|
|
488
|
+
const trackedResourceRef = canonicalGithubPullRequestRef(input.source, number);
|
|
489
|
+
const withCi = input.args?.withCi === true;
|
|
490
|
+
const ciFilter = withCi
|
|
491
|
+
? await buildGithubPrCiFilter(input.source, number, client)
|
|
492
|
+
: null;
|
|
493
|
+
return {
|
|
494
|
+
templateId: "github.pr",
|
|
495
|
+
sources: [
|
|
496
|
+
{
|
|
497
|
+
logicalName: "repo_events",
|
|
498
|
+
sourceType: "github_repo",
|
|
499
|
+
sourceKey: `${config.owner}/${config.repo}`,
|
|
500
|
+
configRef: input.source.configRef ?? null,
|
|
501
|
+
config: githubFollowSourceConfig(input.source),
|
|
502
|
+
},
|
|
503
|
+
...(withCi
|
|
504
|
+
? [{
|
|
505
|
+
logicalName: "ci_runs",
|
|
506
|
+
sourceType: "github_repo_ci",
|
|
507
|
+
sourceKey: `${config.owner}/${config.repo}`,
|
|
508
|
+
configRef: input.source.configRef ?? null,
|
|
509
|
+
config: githubFollowSourceConfig(input.source),
|
|
510
|
+
}]
|
|
511
|
+
: []),
|
|
512
|
+
],
|
|
513
|
+
subscriptions: [
|
|
514
|
+
{
|
|
515
|
+
sourceLogicalName: "repo_events",
|
|
516
|
+
filter: {
|
|
517
|
+
metadata: {
|
|
518
|
+
number,
|
|
519
|
+
isPullRequest: true,
|
|
520
|
+
},
|
|
521
|
+
},
|
|
522
|
+
trackedResourceRef,
|
|
523
|
+
cleanupPolicy: { mode: "on_terminal" },
|
|
524
|
+
},
|
|
525
|
+
...(withCi
|
|
526
|
+
? [{
|
|
527
|
+
sourceLogicalName: "ci_runs",
|
|
528
|
+
filter: ciFilter ?? undefined,
|
|
529
|
+
trackedResourceRef,
|
|
530
|
+
cleanupPolicy: { mode: "on_terminal" },
|
|
531
|
+
}]
|
|
532
|
+
: []),
|
|
533
|
+
],
|
|
534
|
+
};
|
|
535
|
+
}
|
|
536
|
+
return {
|
|
537
|
+
templateId: "github.issue",
|
|
538
|
+
sources: [
|
|
539
|
+
{
|
|
540
|
+
logicalName: "repo_events",
|
|
541
|
+
sourceType: "github_repo",
|
|
542
|
+
sourceKey: `${config.owner}/${config.repo}`,
|
|
543
|
+
configRef: input.source.configRef ?? null,
|
|
544
|
+
config: githubFollowSourceConfig(input.source),
|
|
545
|
+
},
|
|
546
|
+
],
|
|
547
|
+
subscriptions: [
|
|
548
|
+
{
|
|
549
|
+
sourceLogicalName: "repo_events",
|
|
550
|
+
filter: {
|
|
551
|
+
metadata: {
|
|
552
|
+
number,
|
|
553
|
+
isPullRequest: false,
|
|
554
|
+
},
|
|
555
|
+
},
|
|
556
|
+
trackedResourceRef: canonicalGithubIssueRef(input.source, number),
|
|
557
|
+
cleanupPolicy: { mode: "manual" },
|
|
558
|
+
},
|
|
559
|
+
],
|
|
560
|
+
};
|
|
561
|
+
}
|
|
562
|
+
async function buildGithubPrCiFilter(source, number, client) {
|
|
563
|
+
const config = parseGithubSourceConfig(source);
|
|
564
|
+
const baseExpr = `contains(metadata.pullRequestNumbers, ${number})`;
|
|
565
|
+
try {
|
|
566
|
+
const details = await client.getPullRequestDetails({
|
|
567
|
+
owner: config.owner,
|
|
568
|
+
repo: config.repo,
|
|
569
|
+
pullNumber: number,
|
|
570
|
+
auth: config.uxcAuth,
|
|
571
|
+
});
|
|
572
|
+
if (!details.headRefName) {
|
|
573
|
+
throw new Error(`could not resolve head branch for PR #${number}`);
|
|
574
|
+
}
|
|
575
|
+
return {
|
|
576
|
+
expr: buildGithubPrCiFilterExpr(number, details.headRefName, details.headRepositoryFullName),
|
|
577
|
+
};
|
|
578
|
+
}
|
|
579
|
+
catch {
|
|
580
|
+
return { expr: baseExpr };
|
|
581
|
+
}
|
|
582
|
+
}
|
|
583
|
+
function buildGithubPrCiFilterExpr(number, headRefName, headRepositoryFullName) {
|
|
584
|
+
const base = `contains(metadata.pullRequestNumbers, ${number})`;
|
|
585
|
+
const branchExpr = `metadata.headBranch == ${JSON.stringify(headRefName)}`;
|
|
586
|
+
if (!headRepositoryFullName) {
|
|
587
|
+
return `${base} || ${branchExpr}`;
|
|
588
|
+
}
|
|
589
|
+
return [
|
|
590
|
+
base,
|
|
591
|
+
"||",
|
|
592
|
+
"(",
|
|
593
|
+
branchExpr,
|
|
594
|
+
"&&",
|
|
595
|
+
"(",
|
|
596
|
+
`metadata.headRepositoryFullName == ${JSON.stringify(headRepositoryFullName)}`,
|
|
597
|
+
"||",
|
|
598
|
+
"!exists(metadata.headRepositoryFullName)",
|
|
599
|
+
")",
|
|
600
|
+
")",
|
|
601
|
+
].join(" ");
|
|
602
|
+
}
|
|
397
603
|
function projectGithubLifecycleSignal(rawPayload, source) {
|
|
398
604
|
const eventType = asString(rawPayload.type);
|
|
399
605
|
if (eventType !== "PullRequestEvent") {
|
|
@@ -416,6 +622,12 @@ function projectGithubLifecycleSignal(rawPayload, source) {
|
|
|
416
622
|
result: merged ? "merged" : "closed",
|
|
417
623
|
};
|
|
418
624
|
}
|
|
625
|
+
function githubFollowSourceConfig(source) {
|
|
626
|
+
const config = { ...(source.config ?? {}) };
|
|
627
|
+
delete config.number;
|
|
628
|
+
delete config.withCi;
|
|
629
|
+
return config;
|
|
630
|
+
}
|
|
419
631
|
function buildGithubDeliveryHandle(config, eventType, number, comment) {
|
|
420
632
|
if (!number) {
|
|
421
633
|
return null;
|
|
@@ -7,6 +7,7 @@ const uxc_daemon_client_1 = require("@holon-run/uxc-daemon-client");
|
|
|
7
7
|
exports.GITHUB_CI_ENDPOINT = "https://api.github.com";
|
|
8
8
|
exports.DEFAULT_GITHUB_CI_POLL_INTERVAL_SECS = 30;
|
|
9
9
|
exports.DEFAULT_GITHUB_CI_PER_PAGE = 20;
|
|
10
|
+
const MAX_PAGES_PER_SYNC = 10;
|
|
10
11
|
const MAX_SEEN_KEYS = 512;
|
|
11
12
|
const MAX_ERROR_BACKOFF_MULTIPLIER = 8;
|
|
12
13
|
class GithubActionsUxcClient {
|
|
@@ -14,11 +15,12 @@ class GithubActionsUxcClient {
|
|
|
14
15
|
constructor(client = new uxc_daemon_client_1.UxcDaemonClient({ env: process.env })) {
|
|
15
16
|
this.client = client;
|
|
16
17
|
}
|
|
17
|
-
async listWorkflowRuns(config) {
|
|
18
|
+
async listWorkflowRuns(config, page = 1) {
|
|
18
19
|
const payload = {
|
|
19
20
|
owner: config.owner,
|
|
20
21
|
repo: config.repo,
|
|
21
22
|
per_page: config.perPage ?? exports.DEFAULT_GITHUB_CI_PER_PAGE,
|
|
23
|
+
page,
|
|
22
24
|
};
|
|
23
25
|
if (config.eventFilter) {
|
|
24
26
|
payload.event = config.eventFilter;
|
|
@@ -153,30 +155,48 @@ class GithubCiSourceRuntime {
|
|
|
153
155
|
}
|
|
154
156
|
this.lastPollAt.set(sourceId, Date.now());
|
|
155
157
|
const checkpoint = parseGithubCiCheckpoint(source.checkpoint);
|
|
156
|
-
const runs = await this.client.listWorkflowRuns(config);
|
|
157
158
|
let appended = 0;
|
|
158
159
|
let deduped = 0;
|
|
159
|
-
let
|
|
160
|
+
let eventsRead = 0;
|
|
161
|
+
let pageCount = 0;
|
|
162
|
+
const checkpointLastSeenUpdatedAt = checkpoint.lastSeenUpdatedAt ?? undefined;
|
|
163
|
+
let lastSeenUpdatedAt = checkpointLastSeenUpdatedAt;
|
|
160
164
|
const seenKeys = new Set(checkpoint.seenRunKeys ?? []);
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
+
const perPage = config.perPage ?? exports.DEFAULT_GITHUB_CI_PER_PAGE;
|
|
166
|
+
for (let page = 1; page <= MAX_PAGES_PER_SYNC; page += 1) {
|
|
167
|
+
const runs = await this.client.listWorkflowRuns(config, page);
|
|
168
|
+
pageCount = page;
|
|
169
|
+
eventsRead += runs.length;
|
|
170
|
+
let pageHasNewRuns = false;
|
|
171
|
+
for (const run of runs.slice().reverse()) {
|
|
172
|
+
const normalized = normalizeGithubWorkflowRunEvent(source, config, run);
|
|
173
|
+
if (!normalized) {
|
|
174
|
+
continue;
|
|
175
|
+
}
|
|
176
|
+
const eventUpdatedAt = normalized.metadata?.updatedAt;
|
|
177
|
+
const runKey = `${normalized.sourceNativeId}:${normalized.eventVariant}`;
|
|
178
|
+
const isNewerThanCheckpoint = typeof eventUpdatedAt === "string" &&
|
|
179
|
+
(!checkpointLastSeenUpdatedAt || eventUpdatedAt > checkpointLastSeenUpdatedAt);
|
|
180
|
+
const isAtCheckpointBoundary = typeof eventUpdatedAt === "string" &&
|
|
181
|
+
typeof checkpointLastSeenUpdatedAt === "string" &&
|
|
182
|
+
eventUpdatedAt === checkpointLastSeenUpdatedAt;
|
|
183
|
+
const shouldProcess = !checkpointLastSeenUpdatedAt ||
|
|
184
|
+
isNewerThanCheckpoint ||
|
|
185
|
+
(isAtCheckpointBoundary && !seenKeys.has(runKey));
|
|
186
|
+
if (!shouldProcess) {
|
|
187
|
+
continue;
|
|
188
|
+
}
|
|
189
|
+
pageHasNewRuns = true;
|
|
190
|
+
const result = await this.appendSourceEvent(normalized);
|
|
191
|
+
appended += result.appended;
|
|
192
|
+
deduped += result.deduped;
|
|
193
|
+
seenKeys.add(runKey);
|
|
194
|
+
if (typeof eventUpdatedAt === "string" && (!lastSeenUpdatedAt || eventUpdatedAt >= lastSeenUpdatedAt)) {
|
|
195
|
+
lastSeenUpdatedAt = eventUpdatedAt;
|
|
196
|
+
}
|
|
165
197
|
}
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
const shouldProcess = !lastSeenUpdatedAt ||
|
|
169
|
-
(typeof eventUpdatedAt === "string" && eventUpdatedAt > lastSeenUpdatedAt) ||
|
|
170
|
-
!seenKeys.has(runKey);
|
|
171
|
-
if (!shouldProcess) {
|
|
172
|
-
continue;
|
|
173
|
-
}
|
|
174
|
-
const result = await this.appendSourceEvent(normalized);
|
|
175
|
-
appended += result.appended;
|
|
176
|
-
deduped += result.deduped;
|
|
177
|
-
seenKeys.add(runKey);
|
|
178
|
-
if (typeof eventUpdatedAt === "string" && (!lastSeenUpdatedAt || eventUpdatedAt >= lastSeenUpdatedAt)) {
|
|
179
|
-
lastSeenUpdatedAt = eventUpdatedAt;
|
|
198
|
+
if (!pageHasNewRuns || runs.length < perPage) {
|
|
199
|
+
break;
|
|
180
200
|
}
|
|
181
201
|
}
|
|
182
202
|
this.store.updateSourceRuntime(sourceId, {
|
|
@@ -195,8 +215,8 @@ class GithubCiSourceRuntime {
|
|
|
195
215
|
sourceType: "github_repo_ci",
|
|
196
216
|
appended,
|
|
197
217
|
deduped,
|
|
198
|
-
eventsRead
|
|
199
|
-
note: `workflow runs fetched=${
|
|
218
|
+
eventsRead,
|
|
219
|
+
note: `workflow runs fetched=${eventsRead} pages=${pageCount}`,
|
|
200
220
|
};
|
|
201
221
|
}
|
|
202
222
|
catch (error) {
|
|
@@ -240,6 +260,7 @@ function normalizeGithubWorkflowRunEvent(source, config, raw) {
|
|
|
240
260
|
const variant = buildWorkflowRunVariant(workflowName, status, conclusion);
|
|
241
261
|
const actor = asRecord(run.actor);
|
|
242
262
|
const headCommit = asRecord(run.head_commit);
|
|
263
|
+
const headRepository = asRecord(run.head_repository);
|
|
243
264
|
const pullRequestNumbers = extractPullRequestNumbers(run.pull_requests);
|
|
244
265
|
return {
|
|
245
266
|
sourceId: source.sourceId,
|
|
@@ -260,6 +281,7 @@ function normalizeGithubWorkflowRunEvent(source, config, raw) {
|
|
|
260
281
|
event: asString(run.event),
|
|
261
282
|
headSha: asString(run.head_sha),
|
|
262
283
|
headBranch: asString(run.head_branch),
|
|
284
|
+
headRepositoryFullName: asString(headRepository.full_name),
|
|
263
285
|
pullRequestNumbers,
|
|
264
286
|
runNumber: asNumber(run.run_number),
|
|
265
287
|
runAttempt: asNumber(run.run_attempt),
|
|
@@ -279,6 +301,9 @@ function normalizeGithubWorkflowRunEvent(source, config, raw) {
|
|
|
279
301
|
event: asString(run.event),
|
|
280
302
|
head_sha: asString(run.head_sha),
|
|
281
303
|
head_branch: asString(run.head_branch),
|
|
304
|
+
head_repository: {
|
|
305
|
+
full_name: asString(headRepository.full_name),
|
|
306
|
+
},
|
|
282
307
|
pull_requests: pullRequestNumbers.map((number) => ({ number })),
|
|
283
308
|
run_number: asNumber(run.run_number),
|
|
284
309
|
run_attempt: asNumber(run.run_attempt),
|
|
@@ -40,6 +40,12 @@ class RpcUxcRemoteSourceClient {
|
|
|
40
40
|
async sourceStop(namespace, sourceKey) {
|
|
41
41
|
await this.client.sourceStop(namespace, sourceKey);
|
|
42
42
|
}
|
|
43
|
+
sourceStatus(namespace, sourceKey) {
|
|
44
|
+
return this.client.sourceStatus(namespace, sourceKey);
|
|
45
|
+
}
|
|
46
|
+
sourceList() {
|
|
47
|
+
return this.client.sourceList();
|
|
48
|
+
}
|
|
43
49
|
async sourceDelete(namespace, sourceKey) {
|
|
44
50
|
await this.client.sourceDelete(namespace, sourceKey);
|
|
45
51
|
}
|
|
@@ -131,6 +137,8 @@ class RemoteSourceRuntime {
|
|
|
131
137
|
this.errorCounts.delete(sourceId);
|
|
132
138
|
this.nextRetryAt.delete(sourceId);
|
|
133
139
|
const checkpoint = parseRemoteCheckpoint(source.checkpoint);
|
|
140
|
+
const pausedAt = (0, util_1.nowIso)();
|
|
141
|
+
const baseRuntime = checkpoint.managedRuntime ?? checkpointRuntimeFromSource(source, checkpoint) ?? {};
|
|
134
142
|
this.store.updateSourceRuntime(sourceId, {
|
|
135
143
|
status: "paused",
|
|
136
144
|
checkpoint: JSON.stringify({
|
|
@@ -142,6 +150,13 @@ class RemoteSourceRuntime {
|
|
|
142
150
|
streamId: checkpoint.managedSource.streamId,
|
|
143
151
|
},
|
|
144
152
|
} : {}),
|
|
153
|
+
managedRuntime: {
|
|
154
|
+
...baseRuntime,
|
|
155
|
+
status: "stopped",
|
|
156
|
+
updatedAt: pausedAt,
|
|
157
|
+
stoppedAt: pausedAt,
|
|
158
|
+
lastError: null,
|
|
159
|
+
},
|
|
145
160
|
lastError: null,
|
|
146
161
|
}),
|
|
147
162
|
});
|
|
@@ -167,6 +182,47 @@ class RemoteSourceRuntime {
|
|
|
167
182
|
erroredSourceIds: Array.from(this.errorCounts.keys()).sort(),
|
|
168
183
|
};
|
|
169
184
|
}
|
|
185
|
+
async getSourceRuntime(source) {
|
|
186
|
+
if (!REMOTE_SOURCE_TYPES.has(source.sourceType)) {
|
|
187
|
+
return null;
|
|
188
|
+
}
|
|
189
|
+
const binding = managedBindingForSource(source);
|
|
190
|
+
try {
|
|
191
|
+
const managed = await this.client.sourceStatus(binding.namespace, binding.sourceKey);
|
|
192
|
+
return runtimeStateFromManagedView(managed, "live");
|
|
193
|
+
}
|
|
194
|
+
catch {
|
|
195
|
+
return runtimeStateFromCheckpoint(source);
|
|
196
|
+
}
|
|
197
|
+
}
|
|
198
|
+
async listSourceRuntimes(sources) {
|
|
199
|
+
const result = new Map();
|
|
200
|
+
const remoteSources = sources.filter((source) => REMOTE_SOURCE_TYPES.has(source.sourceType));
|
|
201
|
+
for (const source of sources) {
|
|
202
|
+
if (!REMOTE_SOURCE_TYPES.has(source.sourceType)) {
|
|
203
|
+
result.set(source.sourceId, null);
|
|
204
|
+
}
|
|
205
|
+
}
|
|
206
|
+
if (remoteSources.length === 0) {
|
|
207
|
+
return result;
|
|
208
|
+
}
|
|
209
|
+
try {
|
|
210
|
+
const managedSources = await this.client.sourceList();
|
|
211
|
+
const byBinding = new Map(managedSources.map((entry) => [managedSourceBindingKey(entry.namespace, entry.source_key), entry]));
|
|
212
|
+
for (const source of remoteSources) {
|
|
213
|
+
const binding = managedBindingForSource(source);
|
|
214
|
+
const live = byBinding.get(managedSourceBindingKey(binding.namespace, binding.sourceKey));
|
|
215
|
+
result.set(source.sourceId, live ? runtimeStateFromManagedSourceListEntry(live, "live") : runtimeStateFromCheckpoint(source));
|
|
216
|
+
}
|
|
217
|
+
return result;
|
|
218
|
+
}
|
|
219
|
+
catch {
|
|
220
|
+
for (const source of remoteSources) {
|
|
221
|
+
result.set(source.sourceId, runtimeStateFromCheckpoint(source));
|
|
222
|
+
}
|
|
223
|
+
return result;
|
|
224
|
+
}
|
|
225
|
+
}
|
|
170
226
|
async projectLifecycleSignal(source, rawPayload) {
|
|
171
227
|
if (!REMOTE_SOURCE_TYPES.has(source.sourceType)) {
|
|
172
228
|
return null;
|
|
@@ -185,13 +241,37 @@ class RemoteSourceRuntime {
|
|
|
185
241
|
if (typeof module.expandSubscriptionShortcut !== "function") {
|
|
186
242
|
return null;
|
|
187
243
|
}
|
|
188
|
-
const expanded = module.expandSubscriptionShortcut({
|
|
244
|
+
const expanded = await module.expandSubscriptionShortcut({
|
|
189
245
|
name: input.name,
|
|
190
246
|
args: input.args,
|
|
191
247
|
source: moduleInputSource(source),
|
|
192
248
|
});
|
|
193
249
|
return normalizeExpandedSubscriptionPlan(expanded);
|
|
194
250
|
}
|
|
251
|
+
async listFollowTemplates(source) {
|
|
252
|
+
if (!REMOTE_SOURCE_TYPES.has(source.sourceType)) {
|
|
253
|
+
return [];
|
|
254
|
+
}
|
|
255
|
+
const module = await this.moduleRegistry.resolve(source, this.homeDir);
|
|
256
|
+
if (typeof module.listFollowTemplates !== "function") {
|
|
257
|
+
return [];
|
|
258
|
+
}
|
|
259
|
+
return module.listFollowTemplates(moduleInputSource(source));
|
|
260
|
+
}
|
|
261
|
+
async expandFollowTemplate(source, input) {
|
|
262
|
+
if (!REMOTE_SOURCE_TYPES.has(source.sourceType)) {
|
|
263
|
+
return null;
|
|
264
|
+
}
|
|
265
|
+
const module = await this.moduleRegistry.resolve(source, this.homeDir);
|
|
266
|
+
if (typeof module.expandFollowTemplate !== "function") {
|
|
267
|
+
return null;
|
|
268
|
+
}
|
|
269
|
+
return module.expandFollowTemplate({
|
|
270
|
+
template: input.template,
|
|
271
|
+
args: input.args,
|
|
272
|
+
source: moduleInputSource(source),
|
|
273
|
+
});
|
|
274
|
+
}
|
|
195
275
|
async deriveInlinePreview(source, item) {
|
|
196
276
|
if (!REMOTE_SOURCE_TYPES.has(source.sourceType)) {
|
|
197
277
|
return null;
|
|
@@ -331,6 +411,7 @@ class RemoteSourceRuntime {
|
|
|
331
411
|
streamCursor: {
|
|
332
412
|
afterOffset: batch.next_after_offset,
|
|
333
413
|
},
|
|
414
|
+
managedRuntime: checkpointRuntimeFromEnsureResponse(ensured, checkpoint.managedRuntime ?? null, (0, util_1.nowIso)()),
|
|
334
415
|
lastEventAt: (0, util_1.nowIso)(),
|
|
335
416
|
lastError: null,
|
|
336
417
|
}),
|
|
@@ -357,6 +438,7 @@ class RemoteSourceRuntime {
|
|
|
357
438
|
status: source.status === "paused" ? "paused" : "error",
|
|
358
439
|
checkpoint: JSON.stringify({
|
|
359
440
|
...checkpoint,
|
|
441
|
+
managedRuntime: checkpointRuntimeWithError(checkpoint.managedRuntime ?? checkpointRuntimeFromSource(source, checkpoint), error instanceof Error ? error.message : String(error), (0, util_1.nowIso)()),
|
|
360
442
|
lastError: error instanceof Error ? error.message : String(error),
|
|
361
443
|
}),
|
|
362
444
|
});
|
|
@@ -419,6 +501,9 @@ function managedBindingForSource(source) {
|
|
|
419
501
|
sourceKey: checkpoint.managedSource?.sourceKey ?? defaultBinding.sourceKey,
|
|
420
502
|
};
|
|
421
503
|
}
|
|
504
|
+
function managedSourceBindingKey(namespace, sourceKey) {
|
|
505
|
+
return `${namespace}:${sourceKey}`;
|
|
506
|
+
}
|
|
422
507
|
function parseRemoteCheckpoint(checkpoint) {
|
|
423
508
|
if (!checkpoint) {
|
|
424
509
|
return {};
|
|
@@ -430,6 +515,103 @@ function parseRemoteCheckpoint(checkpoint) {
|
|
|
430
515
|
return {};
|
|
431
516
|
}
|
|
432
517
|
}
|
|
518
|
+
function runtimeStateFromManagedView(managed, observation) {
|
|
519
|
+
return {
|
|
520
|
+
backend: "uxc",
|
|
521
|
+
observation,
|
|
522
|
+
namespace: managed.namespace,
|
|
523
|
+
sourceKey: managed.source_key,
|
|
524
|
+
status: managed.status,
|
|
525
|
+
runId: managed.run_id,
|
|
526
|
+
streamId: managed.stream_id,
|
|
527
|
+
lastError: managed.last_error ?? null,
|
|
528
|
+
updatedAt: unixToIso(managed.updated_at_unix),
|
|
529
|
+
startedAt: unixToIso(managed.started_at_unix ?? null),
|
|
530
|
+
stoppedAt: unixToIso(managed.stopped_at_unix ?? null),
|
|
531
|
+
};
|
|
532
|
+
}
|
|
533
|
+
function runtimeStateFromManagedSourceListEntry(managed, observation) {
|
|
534
|
+
return {
|
|
535
|
+
backend: "uxc",
|
|
536
|
+
observation,
|
|
537
|
+
namespace: managed.namespace,
|
|
538
|
+
sourceKey: managed.source_key,
|
|
539
|
+
status: managed.status,
|
|
540
|
+
runId: managed.run_id,
|
|
541
|
+
streamId: managed.stream_id,
|
|
542
|
+
lastError: managed.last_error ?? null,
|
|
543
|
+
updatedAt: unixToIso(managed.updated_at_unix),
|
|
544
|
+
startedAt: null,
|
|
545
|
+
stoppedAt: null,
|
|
546
|
+
};
|
|
547
|
+
}
|
|
548
|
+
function runtimeStateFromCheckpoint(source) {
|
|
549
|
+
const checkpoint = parseRemoteCheckpoint(source.checkpoint);
|
|
550
|
+
const binding = managedBindingForSource(source);
|
|
551
|
+
const runtime = checkpoint.managedRuntime;
|
|
552
|
+
const streamId = runtime?.streamId ?? checkpoint.managedSource?.streamId ?? null;
|
|
553
|
+
if (!runtime && !checkpoint.managedSource) {
|
|
554
|
+
return null;
|
|
555
|
+
}
|
|
556
|
+
return {
|
|
557
|
+
backend: "uxc",
|
|
558
|
+
observation: "cached",
|
|
559
|
+
namespace: binding.namespace,
|
|
560
|
+
sourceKey: binding.sourceKey,
|
|
561
|
+
status: runtime?.status ?? null,
|
|
562
|
+
runId: runtime?.runId ?? null,
|
|
563
|
+
streamId,
|
|
564
|
+
lastError: runtime?.lastError ?? checkpoint.lastError ?? null,
|
|
565
|
+
updatedAt: runtime?.updatedAt ?? source.updatedAt,
|
|
566
|
+
startedAt: runtime?.startedAt ?? null,
|
|
567
|
+
stoppedAt: runtime?.stoppedAt ?? null,
|
|
568
|
+
};
|
|
569
|
+
}
|
|
570
|
+
function checkpointRuntimeFromEnsureResponse(ensured, previous, observedAt) {
|
|
571
|
+
return {
|
|
572
|
+
status: ensured.status,
|
|
573
|
+
runId: ensured.run_id,
|
|
574
|
+
streamId: ensured.stream_id,
|
|
575
|
+
lastError: null,
|
|
576
|
+
updatedAt: observedAt,
|
|
577
|
+
startedAt: previous?.runId === ensured.run_id ? previous.startedAt ?? null : null,
|
|
578
|
+
stoppedAt: null,
|
|
579
|
+
};
|
|
580
|
+
}
|
|
581
|
+
function checkpointRuntimeWithError(previous, lastError, observedAt) {
|
|
582
|
+
return {
|
|
583
|
+
status: previous?.status ?? null,
|
|
584
|
+
runId: previous?.runId ?? null,
|
|
585
|
+
streamId: previous?.streamId ?? null,
|
|
586
|
+
lastError,
|
|
587
|
+
updatedAt: observedAt,
|
|
588
|
+
startedAt: previous?.startedAt ?? null,
|
|
589
|
+
stoppedAt: previous?.stoppedAt ?? null,
|
|
590
|
+
};
|
|
591
|
+
}
|
|
592
|
+
function checkpointRuntimeFromSource(source, checkpoint = parseRemoteCheckpoint(source.checkpoint)) {
|
|
593
|
+
if (checkpoint.managedRuntime) {
|
|
594
|
+
return checkpoint.managedRuntime;
|
|
595
|
+
}
|
|
596
|
+
if (!checkpoint.managedSource) {
|
|
597
|
+
return null;
|
|
598
|
+
}
|
|
599
|
+
return {
|
|
600
|
+
status: null,
|
|
601
|
+
runId: null,
|
|
602
|
+
streamId: checkpoint.managedSource.streamId,
|
|
603
|
+
lastError: checkpoint.lastError ?? null,
|
|
604
|
+
updatedAt: source.updatedAt,
|
|
605
|
+
startedAt: null,
|
|
606
|
+
stoppedAt: null,
|
|
607
|
+
};
|
|
608
|
+
}
|
|
609
|
+
function unixToIso(value) {
|
|
610
|
+
if (typeof value !== "number" || !Number.isFinite(value) || value <= 0) {
|
|
611
|
+
return null;
|
|
612
|
+
}
|
|
613
|
+
return new Date(value * 1000).toISOString();
|
|
614
|
+
}
|
|
433
615
|
function computeErrorBackoffMs(baseIntervalSecs, errorCount) {
|
|
434
616
|
const baseMs = Math.max(1, baseIntervalSecs) * 1000;
|
|
435
617
|
const multiplier = Math.min(2 ** Math.max(0, errorCount - 1), MAX_ERROR_BACKOFF_MULTIPLIER);
|