@lobu/runtime-device 21.0.0 → 21.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/component.json +1 -1
- package/dist/dependencies.bun.lock +5 -5
- package/npm-shrinkwrap.json +23 -23
- package/package.json +1 -1
- package/vendor/cli/dist/commands/_lib/apply/apply-cmd.js +0 -4
- package/vendor/cli/dist/commands/_lib/apply/client.d.ts +1 -4
- package/vendor/cli/dist/commands/_lib/apply/client.js +2 -8
- package/vendor/cli/dist/commands/_lib/apply/deployment.js +0 -1
- package/vendor/cli/dist/commands/_lib/apply/desired-state.d.ts +0 -2
- package/vendor/cli/dist/commands/_lib/apply/diff.d.ts +0 -1
- package/vendor/cli/dist/commands/_lib/apply/diff.js +0 -6
- package/vendor/cli/dist/commands/_lib/view-bundler.js +6 -2
- package/vendor/cli/dist/commands/memory/_lib/schema.d.ts +0 -1
- package/vendor/cli/dist/commands/memory/_lib/schema.js +0 -9
- package/vendor/cli/dist/commands/memory/_lib/seed-cmd.js +0 -2
- package/vendor/cli/dist/connectors/github.ts +269 -194
- package/vendor/cli/dist/connectors/google_calendar.ts +36 -132
- package/vendor/cli/dist/connectors/jira.ts +13 -4
- package/vendor/cli/dist/connectors/producthunt.ts +3 -1
- package/vendor/cli/dist/connectors/reddit.ts +34 -4
- package/vendor/cli/dist/providers.json +0 -1
- package/vendor/cli/dist/runtime-components.json +4 -4
- package/vendor/cli/package.json +1 -1
- package/vendor/connector-sdk/dist/connector-types.d.ts +0 -2
- package/vendor/connector-sdk/package.json +1 -1
- package/vendor/connector-worker/dist/daemon/client.d.ts +4 -1
- package/vendor/connector-worker/dist/daemon/client.js +3 -0
- package/vendor/connector-worker/dist/daemon/executor.js +4 -0
- package/vendor/connector-worker/dist/daemon/start.js +2 -2
- package/vendor/connector-worker/dist/executor/interface.d.ts +7 -1
- package/vendor/connector-worker/dist/executor/isolate.js +40 -0
- package/vendor/connector-worker/package.json +1 -1
- package/vendor/core/dist/contracts/tools/manage-auth-profiles.d.ts +14 -0
- package/vendor/core/dist/contracts/tools/manage-auth-profiles.js +12 -1
- package/vendor/core/dist/contracts/tools/manage-automations.d.ts +0 -10
- package/vendor/core/dist/contracts/tools/manage-automations.js +0 -33
- package/vendor/core/dist/contracts/tools/manage-classifiers.d.ts +4 -34
- package/vendor/core/dist/contracts/tools/manage-classifiers.js +13 -44
- package/vendor/core/dist/contracts/tools/manage-entity.d.ts +0 -9
- package/vendor/core/dist/contracts/tools/manage-entity.js +0 -6
- package/vendor/core/dist/contracts/tools/view-attach.d.ts +20 -0
- package/vendor/core/dist/contracts/tools/view-attach.js +45 -0
- package/vendor/core/dist/contracts/worker/protocol.d.ts +17 -0
- package/vendor/core/dist/contracts/worker/protocol.js +12 -1
- package/vendor/core/dist/index.d.ts +1 -1
- package/vendor/core/dist/provider-config-types.d.ts +0 -9
- package/vendor/core/dist/sdk-compat.d.ts +1 -1
- package/vendor/core/package.json +6 -1
|
@@ -12,7 +12,6 @@ import {
|
|
|
12
12
|
createHttpClient,
|
|
13
13
|
type EventEnvelope,
|
|
14
14
|
type HttpClient,
|
|
15
|
-
paginateByCursor,
|
|
16
15
|
type FeedReadContext,
|
|
17
16
|
type FeedReadResult,
|
|
18
17
|
type RuntimeConnectorDefinition,
|
|
@@ -82,7 +81,7 @@ interface CalendarCheckpoint {
|
|
|
82
81
|
*/
|
|
83
82
|
scope?: string;
|
|
84
83
|
/**
|
|
85
|
-
* Unfinished
|
|
84
|
+
* Unfinished traversal: the `events.list` query as first issued (so the
|
|
86
85
|
* `timeMin`/`timeMax` window stays fixed across runs, which the page token
|
|
87
86
|
* requires) plus the token for the page still to fetch. Absent once the
|
|
88
87
|
* traversal has reached the last page.
|
|
@@ -475,22 +474,9 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
475
474
|
return { status: 'complete' };
|
|
476
475
|
};
|
|
477
476
|
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
calendarId,
|
|
482
|
-
checkpoint.sync_token,
|
|
483
|
-
durableChanges
|
|
484
|
-
);
|
|
485
|
-
if (result) return finish(result.events, result.nextSyncToken);
|
|
486
|
-
// The stored token was rejected (see isSyncTokenRejection). Fall through
|
|
487
|
-
// to one complete bootstrap under the current grant — no retry loop. It
|
|
488
|
-
// either succeeds and overwrites the checkpoint with a token minted under
|
|
489
|
-
// that grant, or throws and the error reaches the run record.
|
|
490
|
-
}
|
|
491
|
-
|
|
492
|
-
// Bootstrap: walk the configured window, resuming a parked page if one is
|
|
493
|
-
// in scope, until the last page hands over a durable cursor.
|
|
477
|
+
const syncToken = resumable ? checkpoint.sync_token : undefined;
|
|
478
|
+
// Bootstrap and incremental traversal share the same page commit boundary.
|
|
479
|
+
// An incremental page keeps its original sync token until the final page.
|
|
494
480
|
const timeMin = new Date();
|
|
495
481
|
timeMin.setDate(timeMin.getDate() - lookbackDays);
|
|
496
482
|
const timeMax = new Date();
|
|
@@ -502,29 +488,39 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
502
488
|
// run: a page token is only valid against the window that minted it.
|
|
503
489
|
const params = pending
|
|
504
490
|
? new URLSearchParams(pending.params)
|
|
505
|
-
:
|
|
491
|
+
: syncToken
|
|
506
492
|
? new URLSearchParams({
|
|
507
493
|
maxResults: '250',
|
|
508
|
-
|
|
509
|
-
showDeleted: 'true',
|
|
510
|
-
timeMin: timeMin.toISOString(),
|
|
494
|
+
syncToken,
|
|
495
|
+
...(durableChanges ? { singleEvents: 'true', showDeleted: 'true' } : {}),
|
|
511
496
|
})
|
|
512
|
-
:
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
|
|
517
|
-
|
|
518
|
-
|
|
497
|
+
: durableChanges
|
|
498
|
+
? new URLSearchParams({
|
|
499
|
+
maxResults: '250',
|
|
500
|
+
singleEvents: 'true',
|
|
501
|
+
showDeleted: 'true',
|
|
502
|
+
timeMin: timeMin.toISOString(),
|
|
503
|
+
})
|
|
504
|
+
: new URLSearchParams({
|
|
505
|
+
maxResults: '250',
|
|
506
|
+
orderBy: 'startTime',
|
|
507
|
+
singleEvents: 'true',
|
|
508
|
+
timeMin: timeMin.toISOString(),
|
|
509
|
+
timeMax: timeMax.toISOString(),
|
|
510
|
+
});
|
|
519
511
|
const baseParams = params.toString();
|
|
520
512
|
|
|
521
513
|
/**
|
|
522
|
-
* Mid-traversal: the
|
|
523
|
-
*
|
|
514
|
+
* Mid-traversal: preserve the old sync token and completed watermark,
|
|
515
|
+
* alongside the next page and the exact query that minted it.
|
|
524
516
|
* A run that stops here resumes from `token`.
|
|
525
517
|
*/
|
|
526
518
|
const parked = (token: string) =>
|
|
527
|
-
({
|
|
519
|
+
({
|
|
520
|
+
scope,
|
|
521
|
+
...(syncToken ? { sync_token: syncToken, last_sync_at: checkpoint.last_sync_at } : {}),
|
|
522
|
+
pending: { params: baseParams, page_token: token },
|
|
523
|
+
}) as Record<string, unknown>;
|
|
528
524
|
|
|
529
525
|
let collected = 0;
|
|
530
526
|
let pageToken = pending?.page_token;
|
|
@@ -538,11 +534,13 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
538
534
|
const url = `${this.BASE_URL}/calendars/${encodeURIComponent(calendarId)}/events?${params.toString()}`;
|
|
539
535
|
const response = await http.raw(url);
|
|
540
536
|
if (!response.ok) {
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
537
|
+
const body = await response.text();
|
|
538
|
+
if (syncToken && isSyncTokenRejection(response.status, body)) {
|
|
539
|
+
// Discard both tokens together. Replayed items keep their origin IDs;
|
|
540
|
+
// a failed bootstrap remains retryable without treating it as complete.
|
|
541
|
+
return this.syncFeed({ ...ctx, checkpoint: { scope } });
|
|
542
|
+
}
|
|
543
|
+
throw new Error(`Calendar events.list error (${response.status}): ${body}`);
|
|
546
544
|
}
|
|
547
545
|
|
|
548
546
|
const data = (await response.json()) as CalendarEventListResponse;
|
|
@@ -580,7 +578,7 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
580
578
|
// bound on a window of nothing but filtered-out items, where
|
|
581
579
|
// `collected` never reaches the cap. The traversal continues next run.
|
|
582
580
|
if (
|
|
583
|
-
collected >= maxResults ||
|
|
581
|
+
(!syncToken && collected >= maxResults) ||
|
|
584
582
|
items.length === 0 ||
|
|
585
583
|
page + 1 >= MAX_SYNC_PAGES
|
|
586
584
|
) {
|
|
@@ -625,100 +623,6 @@ export default class GoogleCalendarConnector extends ConnectorRuntime<Record<str
|
|
|
625
623
|
}
|
|
626
624
|
}
|
|
627
625
|
|
|
628
|
-
// -------------------------------------------------------------------------
|
|
629
|
-
// Incremental sync
|
|
630
|
-
// -------------------------------------------------------------------------
|
|
631
|
-
|
|
632
|
-
private async syncWithToken(
|
|
633
|
-
http: HttpClient,
|
|
634
|
-
calendarId: string,
|
|
635
|
-
syncToken: string,
|
|
636
|
-
durableChanges: boolean
|
|
637
|
-
): Promise<{ events: EventEnvelope[]; nextSyncToken?: string } | null> {
|
|
638
|
-
const events: EventEnvelope[] = [];
|
|
639
|
-
let nextSyncToken: string | undefined;
|
|
640
|
-
let unreadPage: string | undefined;
|
|
641
|
-
const seenTokens = new Set<string>();
|
|
642
|
-
// The stored syncToken was rejected (410 expired, or 403 because it was
|
|
643
|
-
// minted under a superseded grant) — abort and let the caller drop it and
|
|
644
|
-
// fall through to a full sync. Signalled out of the generator via this flag.
|
|
645
|
-
let syncTokenRejected = false;
|
|
646
|
-
|
|
647
|
-
const pages = paginateByCursor<CalendarEvent, string>(
|
|
648
|
-
async (pageToken) => {
|
|
649
|
-
if (pageToken) {
|
|
650
|
-
if (seenTokens.has(pageToken)) {
|
|
651
|
-
throw new Error('Google Calendar returned a repeated page token.');
|
|
652
|
-
}
|
|
653
|
-
seenTokens.add(pageToken);
|
|
654
|
-
}
|
|
655
|
-
// Consume every incremental change before advancing the durable token,
|
|
656
|
-
// so the page size is fixed rather than derived from `max_results`,
|
|
657
|
-
// which limits only the historical bootstrap.
|
|
658
|
-
const params = durableChanges
|
|
659
|
-
? new URLSearchParams({
|
|
660
|
-
maxResults: '250',
|
|
661
|
-
syncToken,
|
|
662
|
-
singleEvents: 'true',
|
|
663
|
-
showDeleted: 'true',
|
|
664
|
-
})
|
|
665
|
-
: new URLSearchParams({
|
|
666
|
-
maxResults: '250',
|
|
667
|
-
syncToken,
|
|
668
|
-
});
|
|
669
|
-
if (pageToken) {
|
|
670
|
-
params.set('pageToken', pageToken);
|
|
671
|
-
}
|
|
672
|
-
|
|
673
|
-
const url = `${this.BASE_URL}/calendars/${encodeURIComponent(calendarId)}/events?${params.toString()}`;
|
|
674
|
-
const response = await http.raw(url);
|
|
675
|
-
|
|
676
|
-
if (!response.ok) {
|
|
677
|
-
const body = await response.text();
|
|
678
|
-
if (isSyncTokenRejection(response.status, body)) {
|
|
679
|
-
syncTokenRejected = true;
|
|
680
|
-
return { items: [], nextCursor: undefined };
|
|
681
|
-
}
|
|
682
|
-
throw new Error(`Calendar events.list error (${response.status}): ${body}`);
|
|
683
|
-
}
|
|
684
|
-
|
|
685
|
-
const data = (await response.json()) as CalendarEventListResponse;
|
|
686
|
-
// Capture the trailing nextSyncToken; generator pages until exhausted.
|
|
687
|
-
nextSyncToken = data.nextSyncToken;
|
|
688
|
-
unreadPage = data.nextPageToken;
|
|
689
|
-
return { items: data.items ?? [], nextCursor: data.nextPageToken };
|
|
690
|
-
},
|
|
691
|
-
{ maxPages: MAX_SYNC_PAGES }
|
|
692
|
-
);
|
|
693
|
-
|
|
694
|
-
for await (const items of pages) {
|
|
695
|
-
for (const calEvent of items) {
|
|
696
|
-
const envelope = durableChanges
|
|
697
|
-
? this.calendarEventToChangeEnvelope(calEvent)
|
|
698
|
-
: this.calendarEventToEnvelope(calEvent);
|
|
699
|
-
if (envelope) events.push(envelope);
|
|
700
|
-
}
|
|
701
|
-
}
|
|
702
|
-
|
|
703
|
-
if (syncTokenRejected) return null;
|
|
704
|
-
// The generator stopped at MAX_SYNC_PAGES with a page still unread. Unlike
|
|
705
|
-
// the bootstrap there is nothing to park — an incremental cursor advances
|
|
706
|
-
// only on the last page — so fail the run and keep the stored token, rather
|
|
707
|
-
// than returning a silently partial batch.
|
|
708
|
-
if (unreadPage) {
|
|
709
|
-
throw new Error(
|
|
710
|
-
'Google Calendar incremental traversal exceeded its page bound before completing.'
|
|
711
|
-
);
|
|
712
|
-
}
|
|
713
|
-
if (durableChanges && !nextSyncToken) {
|
|
714
|
-
throw new Error(
|
|
715
|
-
'Google Calendar incremental changes traversal completed without a durable sync token.'
|
|
716
|
-
);
|
|
717
|
-
}
|
|
718
|
-
|
|
719
|
-
return { events, nextSyncToken };
|
|
720
|
-
}
|
|
721
|
-
|
|
722
626
|
// -------------------------------------------------------------------------
|
|
723
627
|
// Actions
|
|
724
628
|
// -------------------------------------------------------------------------
|
|
@@ -554,13 +554,17 @@ export default class JiraConnector extends ConnectorRuntime<JiraCheckpoint, Jira
|
|
|
554
554
|
// window from its first page instead of wedging the feed.
|
|
555
555
|
let staleCursorRecoverable = pageToken !== undefined;
|
|
556
556
|
let found = 0;
|
|
557
|
+
const seenTokens = new Set<string>();
|
|
557
558
|
for (let page = 0; page < this.MAX_PAGES; page++) {
|
|
558
559
|
const params = new URLSearchParams({
|
|
559
560
|
jql,
|
|
560
561
|
maxResults: String(this.PAGE_SIZE),
|
|
561
562
|
fields: ISSUE_FIELDS,
|
|
562
563
|
});
|
|
563
|
-
if (pageToken)
|
|
564
|
+
if (pageToken) {
|
|
565
|
+
seenTokens.add(pageToken);
|
|
566
|
+
params.set('nextPageToken', pageToken);
|
|
567
|
+
}
|
|
564
568
|
let data: JiraSearchResponse;
|
|
565
569
|
try {
|
|
566
570
|
data = await http.json<JiraSearchResponse>(
|
|
@@ -571,6 +575,7 @@ export default class JiraConnector extends ConnectorRuntime<JiraCheckpoint, Jira
|
|
|
571
575
|
if (staleCursorRecoverable && error instanceof HttpStatusError && error.status === 400) {
|
|
572
576
|
staleCursorRecoverable = false;
|
|
573
577
|
pageToken = undefined;
|
|
578
|
+
seenTokens.clear();
|
|
574
579
|
page--;
|
|
575
580
|
continue;
|
|
576
581
|
}
|
|
@@ -586,17 +591,21 @@ export default class JiraConnector extends ConnectorRuntime<JiraCheckpoint, Jira
|
|
|
586
591
|
}
|
|
587
592
|
found += events.length;
|
|
588
593
|
|
|
589
|
-
|
|
590
|
-
// against a degenerate self-referential cursor.
|
|
591
|
-
if (!data.nextPageToken || issues.length === 0) {
|
|
594
|
+
if (!data.nextPageToken) {
|
|
592
595
|
await ctx.commit(events, { last_sync_at: window.end });
|
|
593
596
|
return { status: 'complete', metadata: { items_found: found } };
|
|
594
597
|
}
|
|
598
|
+
if (seenTokens.has(data.nextPageToken)) {
|
|
599
|
+
throw new Error('Jira returned a repeated page token.');
|
|
600
|
+
}
|
|
595
601
|
pageToken = data.nextPageToken;
|
|
596
602
|
await ctx.commit(events, {
|
|
597
603
|
...(checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
|
|
598
604
|
pending: { window, page_token: pageToken },
|
|
599
605
|
});
|
|
606
|
+
if (issues.length === 0) {
|
|
607
|
+
return { status: 'more', metadata: { items_found: found } };
|
|
608
|
+
}
|
|
600
609
|
}
|
|
601
610
|
// The page cap stopped the walk; its committed token resumes it next run.
|
|
602
611
|
return { status: 'more', metadata: { items_found: found } };
|
|
@@ -294,7 +294,9 @@ export default class ProductHuntConnector extends ConnectorRuntime {
|
|
|
294
294
|
const edges = result.data.posts.edges;
|
|
295
295
|
const pageInfo = result.data.posts.pageInfo;
|
|
296
296
|
const nextCursor = edges.length > 0 && pageInfo.hasNextPage ? pageInfo.endCursor : null;
|
|
297
|
-
|
|
297
|
+
// Retain only pending traversal state. Once exhausted, the next sync
|
|
298
|
+
// must start at the newest posts instead of replaying the final page.
|
|
299
|
+
cursor = nextCursor;
|
|
298
300
|
return { items: edges, nextCursor };
|
|
299
301
|
},
|
|
300
302
|
{ maxPages, initialCursor: cursor, delayMs: this.RATE_LIMIT_MS }
|
|
@@ -68,6 +68,7 @@ interface RedditListingResponse {
|
|
|
68
68
|
interface RedditCheckpoint {
|
|
69
69
|
last_timestamp?: string;
|
|
70
70
|
pagination_token?: string;
|
|
71
|
+
pagination_cutoff?: string;
|
|
71
72
|
}
|
|
72
73
|
|
|
73
74
|
// ---------------------------------------------------------------------------
|
|
@@ -273,8 +274,15 @@ export default class RedditConnector extends ConnectorRuntime {
|
|
|
273
274
|
username = await this.resolveUsername(ctx, userAccessToken);
|
|
274
275
|
}
|
|
275
276
|
|
|
276
|
-
const
|
|
277
|
+
const previous = (ctx.checkpoint ?? {}) as RedditCheckpoint;
|
|
278
|
+
const resumeAfter = previous.pagination_token ?? null;
|
|
279
|
+
let cutoffDate = new Date();
|
|
277
280
|
cutoffDate.setDate(cutoffDate.getDate() - lookbackDays);
|
|
281
|
+
// Keep items eligible for the whole bounded sweep, even across sync runs.
|
|
282
|
+
if (resumeAfter && previous.pagination_cutoff) {
|
|
283
|
+
const savedCutoff = new Date(previous.pagination_cutoff);
|
|
284
|
+
if (Number.isFinite(savedCutoff.getTime())) cutoffDate = savedCutoff;
|
|
285
|
+
}
|
|
278
286
|
|
|
279
287
|
const http = createHttpClient({
|
|
280
288
|
token: accessToken,
|
|
@@ -283,6 +291,8 @@ export default class RedditConnector extends ConnectorRuntime {
|
|
|
283
291
|
});
|
|
284
292
|
|
|
285
293
|
const events: EventEnvelope[] = [];
|
|
294
|
+
const seenItems = new Set<string>();
|
|
295
|
+
const seenCursors = new Set<string>();
|
|
286
296
|
let after: string | null = null;
|
|
287
297
|
let reachedCutoff = false;
|
|
288
298
|
|
|
@@ -303,6 +313,12 @@ export default class RedditConnector extends ConnectorRuntime {
|
|
|
303
313
|
listing = await http.get<RedditListingResponse>(url);
|
|
304
314
|
} catch (error) {
|
|
305
315
|
if (error instanceof HttpStatusError) {
|
|
316
|
+
// A saved fullname can disappear from a changing listing. The head
|
|
317
|
+
// was fetched successfully; clear the rejected anchor for next run.
|
|
318
|
+
if (cursor === resumeAfter && cursor && [400, 404].includes(error.status)) {
|
|
319
|
+
after = null;
|
|
320
|
+
return { items: [], nextCursor: null };
|
|
321
|
+
}
|
|
306
322
|
if (error.status === 404) {
|
|
307
323
|
throw new Error('Subreddit or resource not found. Please check the subreddit name.');
|
|
308
324
|
}
|
|
@@ -314,8 +330,19 @@ export default class RedditConnector extends ConnectorRuntime {
|
|
|
314
330
|
}
|
|
315
331
|
|
|
316
332
|
const children = listing.data.children;
|
|
317
|
-
if (
|
|
318
|
-
|
|
333
|
+
if (cursor) seenCursors.add(cursor);
|
|
334
|
+
after = children.length > 0 ? listing.data.after : null;
|
|
335
|
+
// Reject loops (including a provider ignoring `after` and replaying the
|
|
336
|
+
// head). End the sweep so the next run can start with a fresh listing.
|
|
337
|
+
if (after && seenCursors.has(after)) after = null;
|
|
338
|
+
if (cursor === null && after) {
|
|
339
|
+
seenCursors.add(after);
|
|
340
|
+
// Spend one page on fresh arrivals, then resume the unfinished sweep.
|
|
341
|
+
// Arrivals beyond this page are caught by the next full sweep: never
|
|
342
|
+
// use last_timestamp as a high-water mark on these mutable listings.
|
|
343
|
+
after = resumeAfter ?? after;
|
|
344
|
+
}
|
|
345
|
+
return { items: children, nextCursor: after };
|
|
319
346
|
},
|
|
320
347
|
{ maxPages: this.MAX_PAGES, delayMs: this.RATE_LIMIT_MS }
|
|
321
348
|
);
|
|
@@ -334,6 +361,8 @@ export default class RedditConnector extends ConnectorRuntime {
|
|
|
334
361
|
|
|
335
362
|
// Filter deleted/removed items
|
|
336
363
|
if (itemData.author === '[deleted]') continue;
|
|
364
|
+
if (seenItems.has(itemData.name)) continue;
|
|
365
|
+
seenItems.add(itemData.name);
|
|
337
366
|
|
|
338
367
|
// Use actual Reddit API kind (t3=post, t1=comment) instead of config
|
|
339
368
|
const isPost = child.kind === 't3';
|
|
@@ -359,7 +388,8 @@ export default class RedditConnector extends ConnectorRuntime {
|
|
|
359
388
|
|
|
360
389
|
const checkpoint: RedditCheckpoint = {
|
|
361
390
|
last_timestamp: new Date().toISOString(),
|
|
362
|
-
pagination_token: after ?? undefined,
|
|
391
|
+
pagination_token: !reachedCutoff ? (after ?? undefined) : undefined,
|
|
392
|
+
pagination_cutoff: !reachedCutoff && after ? cutoffDate.toISOString() : undefined,
|
|
363
393
|
};
|
|
364
394
|
|
|
365
395
|
await ctx.commit(events, checkpoint as Record<string, unknown>);
|
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
"server": "dist/server.bundle.mjs",
|
|
6
6
|
"embeddingsServer": "dist/embeddings-server.mjs"
|
|
7
7
|
},
|
|
8
|
-
"version": "21.
|
|
8
|
+
"version": "21.1.0",
|
|
9
9
|
"verify": "dist/verify.mjs"
|
|
10
10
|
},
|
|
11
11
|
"device": {
|
|
@@ -16,7 +16,7 @@
|
|
|
16
16
|
"connector": "node_modules/@lobu/cli/dist/commands/connector.js",
|
|
17
17
|
"worker": "node_modules/@lobu/connector-worker/dist/bin.js"
|
|
18
18
|
},
|
|
19
|
-
"version": "21.
|
|
19
|
+
"version": "21.1.0",
|
|
20
20
|
"verify": "dist/verify.mjs"
|
|
21
21
|
},
|
|
22
22
|
"postgres": {
|
|
@@ -24,7 +24,7 @@
|
|
|
24
24
|
"entries": {
|
|
25
25
|
"postgres": "dist/index.mjs"
|
|
26
26
|
},
|
|
27
|
-
"version": "21.
|
|
27
|
+
"version": "21.1.0",
|
|
28
28
|
"verify": "dist/verify.mjs"
|
|
29
29
|
},
|
|
30
30
|
"embeddings": {
|
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
"entries": {
|
|
33
33
|
"embeddings": "dist/embeddings.js"
|
|
34
34
|
},
|
|
35
|
-
"version": "21.
|
|
35
|
+
"version": "21.1.0",
|
|
36
36
|
"verify": "dist/verify.mjs"
|
|
37
37
|
}
|
|
38
38
|
}
|
package/vendor/cli/package.json
CHANGED
|
@@ -1169,8 +1169,6 @@ export interface ContentItem {
|
|
|
1169
1169
|
created_at: string;
|
|
1170
1170
|
occurred_at: string;
|
|
1171
1171
|
content_date?: string;
|
|
1172
|
-
/** Excerpt for highlighted evidence (when filtering by classification value) */
|
|
1173
|
-
excerpt?: string;
|
|
1174
1172
|
/** Search score fields (only present when query is provided) */
|
|
1175
1173
|
similarity?: number;
|
|
1176
1174
|
text_rank?: number;
|
|
@@ -40,6 +40,8 @@ export interface ExecutorClient {
|
|
|
40
40
|
* — multi-replica safe because the wait is Postgres-mediated.
|
|
41
41
|
*/
|
|
42
42
|
dispatchChromeAction(req: DispatchChromeActionRequest): Promise<Record<string, unknown>>;
|
|
43
|
+
/** Spend only the current run's gateway-held HTTP credentials. */
|
|
44
|
+
httpFetch(runId: number, request: ConnectorHttpRequest, signal?: AbortSignal): Promise<ConnectorHttpResponse>;
|
|
43
45
|
/**
|
|
44
46
|
* Post a device-side automation exit report and read the server's decision.
|
|
45
47
|
* `status: "resume"` means the run is still claimed and the caller should
|
|
@@ -67,7 +69,7 @@ export interface ExecutorClient {
|
|
|
67
69
|
* its request-body reads with the same shapes from that file.
|
|
68
70
|
*/
|
|
69
71
|
export type { CompleteActionRequest, CompleteAgentTurnRequest, CompleteAgentTurnResponse, CompleteAuthRequest, CompleteAutomationRequest, CompleteAutomationResponse, CompleteDeviceChatRequest, CompleteDeviceChatResponse, CompleteEmbeddingsRequest, CompleteRequest, ContentItem, DispatchChromeActionRequest, DispatchChromeActionResponse, EmbedEvent, EmitAuthArtifactRequest, HeartbeatRequest, HeartbeatResponse, OAuthCredentials, PollAuthSignalRequest, PollAuthSignalResponse, PollResponse, StreamBatch, } from "@lobu/core/contracts/worker/protocol";
|
|
70
|
-
import type { CompleteActionRequest, CompleteAgentTurnRequest, CompleteAgentTurnResponse, CompleteAuthRequest, CompleteAutomationRequest, CompleteAutomationResponse, CompleteDeviceChatRequest, CompleteDeviceChatResponse, CompleteEmbeddingsRequest, CompleteRequest, DispatchChromeActionRequest, EmbedEvent, EmitAuthArtifactRequest, HeartbeatRequest, HeartbeatResponse, PollAuthSignalRequest, PollAuthSignalResponse, PollResponse, StreamBatch } from "@lobu/core/contracts/worker/protocol";
|
|
72
|
+
import type { ConnectorHttpRequest, ConnectorHttpResponse, CompleteActionRequest, CompleteAgentTurnRequest, CompleteAgentTurnResponse, CompleteAuthRequest, CompleteAutomationRequest, CompleteAutomationResponse, CompleteDeviceChatRequest, CompleteDeviceChatResponse, CompleteEmbeddingsRequest, CompleteRequest, DispatchChromeActionRequest, EmbedEvent, EmitAuthArtifactRequest, HeartbeatRequest, HeartbeatResponse, PollAuthSignalRequest, PollAuthSignalResponse, PollResponse, StreamBatch } from "@lobu/core/contracts/worker/protocol";
|
|
71
73
|
import type { AgentKind } from "@lobu/core/contracts/worker/device-automation";
|
|
72
74
|
/** Capability strings the worker advertises, keyed by name (e.g. `browser.debugger`). */
|
|
73
75
|
export type WorkerCapabilities = Record<string, boolean>;
|
|
@@ -227,6 +229,7 @@ export declare class WorkerClient implements ExecutorClient {
|
|
|
227
229
|
* times out. Throws on failure/timeout with the gateway's error message.
|
|
228
230
|
*/
|
|
229
231
|
dispatchChromeAction(req: DispatchChromeActionRequest): Promise<Record<string, unknown>>;
|
|
232
|
+
httpFetch(runId: number, request: ConnectorHttpRequest, signal?: AbortSignal): Promise<ConnectorHttpResponse>;
|
|
230
233
|
/**
|
|
231
234
|
* Health check
|
|
232
235
|
*/
|
|
@@ -311,6 +311,9 @@ export class WorkerClient {
|
|
|
311
311
|
throw new Error(result.error_message ??
|
|
312
312
|
`Chrome action '${req.action_key}' ${result.status === 'timeout' ? 'timed out' : 'failed'}`);
|
|
313
313
|
}
|
|
314
|
+
async httpFetch(runId, request, signal) {
|
|
315
|
+
return this.requestJson('/api/workers/http-fetch', { worker_id: this.id, run_id: runId, request }, signal);
|
|
316
|
+
}
|
|
314
317
|
/**
|
|
315
318
|
* Health check
|
|
316
319
|
*/
|
|
@@ -282,6 +282,7 @@ async function executeSyncRun(client, job, env, cfg) {
|
|
|
282
282
|
executor: laneExecutor,
|
|
283
283
|
job: {
|
|
284
284
|
mode: 'sync',
|
|
285
|
+
httpAuth: job.http_auth,
|
|
285
286
|
config: mergeEnv(env, job.connection_credentials, feedConfig),
|
|
286
287
|
checkpoint: checkpoint,
|
|
287
288
|
env,
|
|
@@ -292,6 +293,7 @@ async function executeSyncRun(client, job, env, cfg) {
|
|
|
292
293
|
entityIds: job.entity_ids ?? [],
|
|
293
294
|
},
|
|
294
295
|
hooks: {
|
|
296
|
+
...(job.http_auth ? { onHttpFetch: (request, signal) => client.httpFetch(run_id, request, signal) } : {}),
|
|
295
297
|
// One chunk of a `ctx.commit`, sent as one page: the gateway stores its
|
|
296
298
|
// events and checkpoint in one transaction or not at all. Any failure
|
|
297
299
|
// propagates and ends the run; the next run resumes from the last
|
|
@@ -444,6 +446,7 @@ async function executeActionRun(client, job, env, cfg) {
|
|
|
444
446
|
executor: laneExecutor,
|
|
445
447
|
job: {
|
|
446
448
|
mode: 'action',
|
|
449
|
+
httpAuth: job.http_auth,
|
|
447
450
|
// NOTE: unlike the gateway's inline action path (manage_operations
|
|
448
451
|
// `executeLocalActionInline`, which merges the connection's own config
|
|
449
452
|
// so an action can read e.g. a `restaurants_url`), this worker-fleet
|
|
@@ -463,6 +466,7 @@ async function executeActionRun(client, job, env, cfg) {
|
|
|
463
466
|
credentials: credentials ?? null,
|
|
464
467
|
},
|
|
465
468
|
hooks: {
|
|
469
|
+
...(job.http_auth ? { onHttpFetch: (request, signal) => client.httpFetch(run_id, request, signal) } : {}),
|
|
466
470
|
onChromeDispatch: async (actionKey, actionInput) => {
|
|
467
471
|
return client.dispatchChromeAction({
|
|
468
472
|
parent_run_id: run_id,
|
|
@@ -7,7 +7,7 @@ import { randomUUID } from 'node:crypto';
|
|
|
7
7
|
import { createRequire } from 'node:module';
|
|
8
8
|
import { hostname } from 'node:os';
|
|
9
9
|
import { isKnownPlatform } from '@lobu/core';
|
|
10
|
-
import { EXECUTION_BACKENDS, } from '@lobu/core/contracts/worker/protocol';
|
|
10
|
+
import { CONNECTOR_HTTP_AUTH_CAPABILITY, EXECUTION_BACKENDS, } from '@lobu/core/contracts/worker/protocol';
|
|
11
11
|
import { assertExternalDepsResolvable } from '../compile/index.js';
|
|
12
12
|
import { buildConnectorWorkerEnv } from '../env.js';
|
|
13
13
|
import { assertConnectorRuntimeLoadable } from '../self-check/index.js';
|
|
@@ -97,7 +97,7 @@ export async function startDaemonCommand(opts) {
|
|
|
97
97
|
// keep draining under the single-reply contract they were admitted with.
|
|
98
98
|
const workerCapabilities = platform
|
|
99
99
|
? Object.fromEntries(capabilities.map((name) => [name, true]))
|
|
100
|
-
: { db_egress_hardening: true, agent_turn: true, agent_turn_per_input_responses: true };
|
|
100
|
+
: { db_egress_hardening: true, agent_turn: true, agent_turn_per_input_responses: true, [CONNECTOR_HTTP_AUTH_CAPABILITY]: true };
|
|
101
101
|
// Auto-discover device identity from the host when not passed: a device
|
|
102
102
|
// worker defaults to `<platform>:<short-hostname>` and a hostname label. An
|
|
103
103
|
// interactive daemon instead derives its id from the exact inherited
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
import type { AgentTurnEvent, AgentTurnInput, AgentTurnOutput, AgentTurnSteer, RuntimeExecRequest, RuntimeExecResult } from '../agent-turn/types.js';
|
|
2
2
|
import type { AuthResult, ConnectorWebhookSchema, EventEnvelope, SyncCredentials, WebhookRegistration } from '@lobu/connector-sdk';
|
|
3
|
+
import type { ConnectorHttpRequest, ConnectorHttpResponse } from '@lobu/core/contracts/worker/protocol';
|
|
3
4
|
/**
|
|
4
5
|
* Executor mode discriminator. The executor speaks the same V1 SDK shapes
|
|
5
6
|
* the connector code expects: `SyncContext` / `ActionContext` / `AuthContext`
|
|
6
7
|
* in, `SyncResult` / `ActionResult` / `AuthResult` out, no envelope.
|
|
7
8
|
*/
|
|
8
9
|
export type ExecutorJob = {
|
|
10
|
+
/** HTTP credentials stay in the gateway; every fetch uses the host hook. */
|
|
11
|
+
httpAuth?: boolean;
|
|
12
|
+
} & ({
|
|
9
13
|
mode: 'sync';
|
|
10
14
|
feedKey?: string | null;
|
|
11
15
|
/** Feed-instance id (feeds.id) — namespaces emitted origin_ids per feed. */
|
|
@@ -83,7 +87,7 @@ export type ExecutorJob = {
|
|
|
83
87
|
sessionState: Record<string, unknown> | null;
|
|
84
88
|
externalId: string;
|
|
85
89
|
env: Record<string, string | undefined>;
|
|
86
|
-
};
|
|
90
|
+
});
|
|
87
91
|
/**
|
|
88
92
|
* Result shape returned by the executor. One discriminated union per mode
|
|
89
93
|
* mirrors the SDK's `ActionResult` / `AuthResult` directly. Sync is
|
|
@@ -144,6 +148,8 @@ export type ExecutorResult = {
|
|
|
144
148
|
mode: 'webhook_unregister';
|
|
145
149
|
};
|
|
146
150
|
export interface ExecutionHooks {
|
|
151
|
+
/** Gateway-bound HTTP execution. The callback and its authority never enter the guest. */
|
|
152
|
+
onHttpFetch?: (request: ConnectorHttpRequest, signal?: AbortSignal) => Promise<ConnectorHttpResponse>;
|
|
147
153
|
/**
|
|
148
154
|
* Stop the run from outside: the guest is terminated and `execute` rejects
|
|
149
155
|
* with the abort as its error. An agent turn arms this when the gateway's
|
|
@@ -29,6 +29,7 @@ import { decideEgress } from '@lobu/connector-sdk/egress-policy';
|
|
|
29
29
|
import { isReservedIp, stripIpv6Brackets } from '@lobu/connector-sdk/ip-reachability';
|
|
30
30
|
import { normalizeDomainPattern } from '@lobu/core';
|
|
31
31
|
import { CredentialVault } from '../egress/credentials.js';
|
|
32
|
+
import { CONNECTOR_HTTP_MAX_BYTES } from '@lobu/core/contracts/worker/protocol';
|
|
32
33
|
import { EgressDispatcher, fetchPublicUrl, MalformedHostError, parseExemptHosts, PrivateAddressError, resolveEgressAddresses, } from '../egress/transport.js';
|
|
33
34
|
import { IsolateHost, IsolateHostError } from '../isolate/bridge.js';
|
|
34
35
|
import { assertIsolateEligible } from '../isolate/eligibility.js';
|
|
@@ -410,6 +411,9 @@ export class IsolateExecutor {
|
|
|
410
411
|
return ivm;
|
|
411
412
|
}
|
|
412
413
|
async execute(compiledCode, job, hooks) {
|
|
414
|
+
if (job.httpAuth && !hooks?.onHttpFetch) {
|
|
415
|
+
throw new Error('HTTP authentication requires a gateway fetch hook');
|
|
416
|
+
}
|
|
413
417
|
assertIsolateEligible(compiledCode);
|
|
414
418
|
const ivm = await this.requireIsolatedVm();
|
|
415
419
|
const tail = new RingBuffer(STREAM_TAIL_CAP_BYTES);
|
|
@@ -551,6 +555,7 @@ export class IsolateExecutor {
|
|
|
551
555
|
};
|
|
552
556
|
const network = {
|
|
553
557
|
egress, vault, spends: new Set(), log,
|
|
558
|
+
onHttpFetch: job.httpAuth ? hooks?.onHttpFetch : undefined,
|
|
554
559
|
...(job.mode === 'agent_turn' && job.credentials?.accessToken ? {
|
|
555
560
|
agentGateway: {
|
|
556
561
|
origin: new URL(job.turn.provider.baseUrl).origin,
|
|
@@ -1058,6 +1063,41 @@ export class IsolateExecutor {
|
|
|
1058
1063
|
let method = request.method;
|
|
1059
1064
|
let headers = new Headers(request.headers);
|
|
1060
1065
|
let requestBody = body instanceof Uint8Array ? body : body instanceof ArrayBuffer ? new Uint8Array(body) : null;
|
|
1066
|
+
if (net.onHttpFetch) {
|
|
1067
|
+
// Authenticated jobs never dial from the worker or resolve its vault.
|
|
1068
|
+
// The gateway owns credential injection and pinned-address transport.
|
|
1069
|
+
await this.assertHostAllowed('fetch', url.hostname, net.log);
|
|
1070
|
+
if (url.protocol !== 'https:')
|
|
1071
|
+
throw refuse(net.log, 'gateway HTTP authentication requires HTTPS');
|
|
1072
|
+
if (requestBody && requestBody.byteLength > CONNECTOR_HTTP_MAX_BYTES) {
|
|
1073
|
+
throw new RangeError(`fetch request body exceeded the ${CONNECTOR_HTTP_MAX_BYTES}-byte cap`);
|
|
1074
|
+
}
|
|
1075
|
+
const response = await net.onHttpFetch({
|
|
1076
|
+
url: url.href,
|
|
1077
|
+
method,
|
|
1078
|
+
headers: Object.fromEntries(headers.entries()),
|
|
1079
|
+
...(requestBody ? { body: Buffer.from(requestBody).toString('base64') } : {}),
|
|
1080
|
+
}, signal);
|
|
1081
|
+
const limit = Math.min(CONNECTOR_HTTP_MAX_BYTES, this.options.fetchBodyBytes);
|
|
1082
|
+
if (response.body.length > Math.ceil(limit / 3) * 4) {
|
|
1083
|
+
throw new RangeError(`fetch response body exceeded the ${limit}-byte cap`);
|
|
1084
|
+
}
|
|
1085
|
+
const bytes = Buffer.from(response.body, 'base64');
|
|
1086
|
+
if (bytes.byteLength > limit) {
|
|
1087
|
+
throw new RangeError(`fetch response body exceeded the ${limit}-byte cap`);
|
|
1088
|
+
}
|
|
1089
|
+
const responseHeaders = new Headers(response.headers);
|
|
1090
|
+
return {
|
|
1091
|
+
status: response.status,
|
|
1092
|
+
statusText: response.statusText,
|
|
1093
|
+
url: url.href,
|
|
1094
|
+
redirected: false,
|
|
1095
|
+
headers: [...responseHeaders.entries()],
|
|
1096
|
+
body: method === 'HEAD' || [204, 205, 304].includes(response.status)
|
|
1097
|
+
? null
|
|
1098
|
+
: new Response(bytes).body,
|
|
1099
|
+
};
|
|
1100
|
+
}
|
|
1061
1101
|
const redirectMode = request.redirect;
|
|
1062
1102
|
let redirected = false;
|
|
1063
1103
|
/** Headers a placeholder was resolved into: they never follow a redirect off this origin, whatever their name. */
|