@lobu/runtime-device 20.1.0 → 21.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/component.json +1 -1
- package/dist/dependencies.bun.lock +46 -40
- package/npm-shrinkwrap.json +283 -237
- package/package.json +1 -1
- package/vendor/cli/dist/commands/_lib/apply/apply-cmd.d.ts +7 -2
- package/vendor/cli/dist/commands/_lib/apply/apply-cmd.js +77 -75
- package/vendor/cli/dist/commands/_lib/apply/client.d.ts +40 -25
- package/vendor/cli/dist/commands/_lib/apply/client.js +31 -39
- package/vendor/cli/dist/commands/_lib/apply/deployment.d.ts +1 -1
- package/vendor/cli/dist/commands/_lib/apply/deployment.js +2 -2
- package/vendor/cli/dist/commands/_lib/apply/desired-state.d.ts +47 -11
- package/vendor/cli/dist/commands/_lib/apply/desired-state.js +106 -7
- package/vendor/cli/dist/commands/_lib/apply/diff.d.ts +14 -6
- package/vendor/cli/dist/commands/_lib/apply/diff.js +42 -23
- package/vendor/cli/dist/commands/_lib/apply/map-config.js +2 -3
- package/vendor/cli/dist/commands/_lib/apply/render.js +2 -0
- package/vendor/cli/dist/commands/_lib/apply/rollback-cmd.js +3 -3
- package/vendor/cli/dist/commands/_lib/connector-loader.d.ts +5 -1
- package/vendor/cli/dist/commands/_lib/connector-loader.js +1 -1
- package/vendor/cli/dist/commands/_lib/ensure-deps-installed.d.ts +15 -26
- package/vendor/cli/dist/commands/_lib/ensure-deps-installed.js +227 -136
- package/vendor/cli/dist/commands/_lib/view-bundler.d.ts +33 -0
- package/vendor/cli/dist/commands/_lib/view-bundler.js +207 -0
- package/vendor/cli/dist/commands/_lib/view-reapply.d.ts +60 -0
- package/vendor/cli/dist/commands/_lib/view-reapply.js +182 -0
- package/vendor/cli/dist/commands/dev.d.ts +14 -5
- package/vendor/cli/dist/commands/dev.js +56 -8
- package/vendor/cli/dist/commands/init.d.ts +1 -1
- package/vendor/cli/dist/commands/init.js +4 -4
- package/vendor/cli/dist/commands/login.js +3 -0
- package/vendor/cli/dist/config/define.d.ts +25 -8
- package/vendor/cli/dist/config/define.js +24 -0
- package/vendor/cli/dist/connectors/README.md +15 -16
- package/vendor/cli/dist/connectors/chat-user-identity.ts +41 -0
- package/vendor/cli/dist/connectors/github.ts +15 -9
- package/vendor/cli/dist/connectors/google-identity.ts +11 -0
- package/vendor/cli/dist/connectors/google_calendar.ts +21 -31
- package/vendor/cli/dist/connectors/google_drive.ts +106 -131
- package/vendor/cli/dist/connectors/google_gmail.ts +53 -30
- package/vendor/cli/dist/connectors/hackernews.ts +2 -5
- package/vendor/cli/dist/connectors/jira.ts +62 -27
- package/vendor/cli/dist/connectors/linear.ts +55 -21
- package/vendor/cli/dist/connectors/microsoft_outlook.ts +151 -110
- package/vendor/cli/dist/connectors/postgres.ts +19 -11
- package/vendor/cli/dist/connectors/producthunt.ts +8 -6
- package/vendor/cli/dist/connectors/reddit.ts +2 -2
- package/vendor/cli/dist/connectors/rss.ts +2 -2
- package/vendor/cli/dist/connectors/slack-identity.ts +19 -0
- package/vendor/cli/dist/connectors/whatsapp_web.ts +5 -4
- package/vendor/cli/dist/connectors/x.ts +32 -17
- package/vendor/cli/dist/connectors/youtube.ts +18 -5
- package/vendor/cli/dist/internal/connector-compiler.d.ts +1 -1
- package/vendor/cli/dist/internal/connector-compiler.js +216 -17
- package/vendor/cli/dist/internal/credentials.js +5 -1
- package/vendor/cli/dist/internal/oauth.d.ts +15 -1
- package/vendor/cli/dist/internal/oauth.js +9 -1
- package/vendor/cli/dist/runtime-components.json +4 -4
- package/vendor/cli/package.json +2 -1
- package/vendor/connector-sdk/dist/connector-runtime.d.ts +4 -6
- package/vendor/connector-sdk/dist/connector-runtime.js +3 -5
- package/vendor/connector-sdk/dist/connector-types.d.ts +28 -13
- package/vendor/connector-sdk/dist/define-connector.d.ts +4 -1
- package/vendor/connector-sdk/dist/define-connector.js +4 -1
- package/vendor/connector-sdk/dist/http-client.d.ts +18 -7
- package/vendor/connector-sdk/dist/http-client.js +15 -13
- package/vendor/connector-sdk/dist/index.d.ts +1 -1
- package/vendor/connector-sdk/dist/retry.d.ts +15 -3
- package/vendor/connector-sdk/dist/retry.js +40 -64
- package/vendor/connector-sdk/package.json +1 -1
- package/vendor/connector-worker/dist/agent-turn/guest-entry.js +122 -10
- package/vendor/connector-worker/dist/agent-turn/guest.bundle.js +78 -10
- package/vendor/connector-worker/dist/agent-turn/types.d.ts +34 -4
- package/vendor/connector-worker/dist/compile/index.d.ts +21 -2
- package/vendor/connector-worker/dist/compile/index.js +41 -8
- package/vendor/connector-worker/dist/compile/source-capture.d.ts +10 -0
- package/vendor/connector-worker/dist/compile/source-capture.js +108 -0
- package/vendor/connector-worker/dist/daemon/agent-turn.js +97 -10
- package/vendor/connector-worker/dist/daemon/automation-process.d.ts +32 -0
- package/vendor/connector-worker/dist/daemon/automation-process.js +102 -1
- package/vendor/connector-worker/dist/daemon/automation.d.ts +1 -1
- package/vendor/connector-worker/dist/daemon/builtins/index.js +8 -0
- package/vendor/connector-worker/dist/daemon/builtins/os-shell.d.ts +13 -1
- package/vendor/connector-worker/dist/daemon/builtins/os-shell.js +28 -2
- package/vendor/connector-worker/dist/daemon/executor.d.ts +0 -1
- package/vendor/connector-worker/dist/daemon/executor.js +42 -51
- package/vendor/connector-worker/dist/daemon/start.js +5 -1
- package/vendor/connector-worker/dist/executor/interface.d.ts +12 -8
- package/vendor/connector-worker/dist/executor/isolate.js +45 -28
- package/vendor/connector-worker/dist/self-check/index.js +15 -13
- package/vendor/connector-worker/package.json +1 -1
- package/vendor/core/dist/agent-policy.js +3 -0
- package/vendor/core/dist/connector-query-errors.d.ts +3 -3
- package/vendor/core/dist/connector-query-errors.js +8 -3
- package/vendor/core/dist/contracts/tools/manage-connections.d.ts +24 -0
- package/vendor/core/dist/contracts/tools/manage-connections.js +13 -0
- package/vendor/core/dist/contracts/tools/manage-entity-schema.d.ts +15 -25
- package/vendor/core/dist/contracts/tools/manage-entity-schema.js +3 -5
- package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +11 -0
- package/vendor/core/dist/contracts/tools/manage-feeds.js +12 -1
- package/vendor/core/dist/contracts/tools/manage-operations.d.ts +47 -6
- package/vendor/core/dist/contracts/tools/manage-operations.js +36 -1
- package/vendor/core/dist/contracts/tools/manage-views.d.ts +293 -0
- package/vendor/core/dist/contracts/tools/manage-views.js +203 -0
- package/vendor/core/dist/contracts/tools/source-files-schema.d.ts +7 -0
- package/vendor/core/dist/contracts/tools/source-files-schema.js +19 -0
- package/vendor/core/dist/contracts/tools/source-files.d.ts +14 -0
- package/vendor/core/dist/contracts/tools/source-files.js +55 -0
- package/vendor/core/dist/contracts/tools/view-content-hash.d.ts +37 -0
- package/vendor/core/dist/contracts/tools/view-content-hash.js +58 -0
- package/vendor/core/dist/contracts/tools/view-path.d.ts +18 -0
- package/vendor/core/dist/contracts/tools/view-path.js +24 -0
- package/vendor/core/dist/contracts/worker/protocol.d.ts +73 -8
- package/vendor/core/dist/contracts/worker/protocol.js +58 -9
- package/vendor/core/dist/credentials.d.ts +16 -1
- package/vendor/core/dist/credentials.js +10 -1
- package/vendor/core/dist/reserved.d.ts +3 -3
- package/vendor/core/dist/reserved.js +9 -1
- package/vendor/core/dist/utils/secret-redaction.js +2 -0
- package/vendor/core/package.json +25 -5
- package/vendor/core/dist/contracts/tools/manage-view-templates.d.ts +0 -165
- package/vendor/core/dist/contracts/tools/manage-view-templates.js +0 -130
|
@@ -14,9 +14,9 @@ import {
|
|
|
14
14
|
type EventEnvelope,
|
|
15
15
|
fileDownloadOutput,
|
|
16
16
|
type HttpClient,
|
|
17
|
+
HttpStatusError,
|
|
17
18
|
inlineContentBudget,
|
|
18
19
|
inlineMaxBytesSchema,
|
|
19
|
-
paginateByCursor,
|
|
20
20
|
requireBearerClient,
|
|
21
21
|
type SyncContext,
|
|
22
22
|
type SyncResult,
|
|
@@ -89,7 +89,24 @@ interface GraphPagedResponse<T> {
|
|
|
89
89
|
// ---------------------------------------------------------------------------
|
|
90
90
|
|
|
91
91
|
interface OutlookCheckpoint {
|
|
92
|
+
/** When the last complete traversal began; the next window starts here. */
|
|
92
93
|
last_sync_at?: string;
|
|
94
|
+
/**
|
|
95
|
+
* A traversal that stopped at the per-run cap with pages left. `window`
|
|
96
|
+
* is the fixed range it walks and `next_link` Graph's cursor for the first
|
|
97
|
+
* page not yet stored, so the next run continues exactly where this one
|
|
98
|
+
* stopped instead of re-reading the newest page and never reaching the rest.
|
|
99
|
+
*/
|
|
100
|
+
pending?: { window: { start: string; end: string }; next_link: string };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const WINDOW_OVERLAP_MS = 5 * 60_000;
|
|
104
|
+
|
|
105
|
+
/** ISO timestamp `days` from now (negative for the past). */
|
|
106
|
+
function daysFromNow(days: number): string {
|
|
107
|
+
const date = new Date();
|
|
108
|
+
date.setDate(date.getDate() + days);
|
|
109
|
+
return date.toISOString();
|
|
93
110
|
}
|
|
94
111
|
|
|
95
112
|
// ---------------------------------------------------------------------------
|
|
@@ -448,67 +465,53 @@ export default class MicrosoftOutlookConnector extends ConnectorRuntime {
|
|
|
448
465
|
private async syncMessages(ctx: SyncContext, http: HttpClient): Promise<SyncResult> {
|
|
449
466
|
const config = ctx.config as Record<string, unknown>;
|
|
450
467
|
const folder = (config.folder as string) ?? 'inbox';
|
|
451
|
-
const maxResults = (config.max_results as number) ?? 50;
|
|
452
468
|
const lookbackDays = (config.lookback_days as number) ?? 30;
|
|
453
|
-
const
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
469
|
+
const checkpoint = (ctx.checkpoint ?? {}) as OutlookCheckpoint;
|
|
470
|
+
|
|
471
|
+
// A fixed window per traversal: from where the last complete one began
|
|
472
|
+
// (or the lookback) up to when this one began. Mail arriving meanwhile
|
|
473
|
+
// belongs to the next window, so a capped walk can never be pushed past
|
|
474
|
+
// messages it has not reached by newer arrivals landing at the front.
|
|
475
|
+
// The start overlaps the previous window by a few minutes: Graph can
|
|
476
|
+
// surface a message after its receivedDateTime, and `origin_id` makes the
|
|
477
|
+
// replayed boundary a supersede, not a duplicate.
|
|
478
|
+
const window = checkpoint.pending?.window ?? {
|
|
479
|
+
start: checkpoint.last_sync_at
|
|
480
|
+
? new Date(Date.parse(checkpoint.last_sync_at) - WINDOW_OVERLAP_MS).toISOString()
|
|
481
|
+
: daysFromNow(-lookbackDays),
|
|
482
|
+
end: new Date().toISOString(),
|
|
483
|
+
};
|
|
460
484
|
const firstUrl =
|
|
461
|
-
`${this.API_BASE}/me/mailFolders/${
|
|
462
|
-
`?$top=${
|
|
485
|
+
`${this.API_BASE}/me/mailFolders/${encodeURIComponent(folder)}/messages` +
|
|
486
|
+
`?$top=${this.PAGE_SIZE}` +
|
|
463
487
|
'&$orderby=receivedDateTime desc' +
|
|
464
|
-
`&$filter=receivedDateTime ge ${
|
|
488
|
+
`&$filter=receivedDateTime ge ${window.start} and receivedDateTime lt ${window.end}` +
|
|
465
489
|
'&$select=id,conversationId,subject,bodyPreview,from,toRecipients,ccRecipients,receivedDateTime,sentDateTime,hasAttachments,importance,isRead,webLink';
|
|
466
490
|
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
to: formatRecipients(msg.toRecipients ?? []),
|
|
492
|
-
cc: formatRecipients(msg.ccRecipients ?? []),
|
|
493
|
-
importance: msg.importance,
|
|
494
|
-
has_attachments: msg.hasAttachments,
|
|
495
|
-
is_read: msg.isRead,
|
|
496
|
-
},
|
|
497
|
-
});
|
|
498
|
-
fetched++;
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
if (ctx.emitEvents) await ctx.emitEvents(events.splice(0));
|
|
502
|
-
|
|
503
|
-
if (fetched >= maxResults) break;
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
return {
|
|
507
|
-
events,
|
|
508
|
-
checkpoint: {
|
|
509
|
-
last_sync_at: new Date().toISOString(),
|
|
510
|
-
} satisfies OutlookCheckpoint as Record<string, unknown>,
|
|
511
|
-
};
|
|
491
|
+
return this.walkPages<GraphMessage>(ctx, http, {
|
|
492
|
+
window,
|
|
493
|
+
firstUrl,
|
|
494
|
+
maxResults: (config.max_results as number) ?? 50,
|
|
495
|
+
toEvent: (msg) => ({
|
|
496
|
+
origin_id: `outlook_msg_${msg.id}`,
|
|
497
|
+
title: msg.subject,
|
|
498
|
+
payload_text: msg.bodyPreview || msg.subject,
|
|
499
|
+
author_name: msg.from?.emailAddress?.name || msg.from?.emailAddress?.address,
|
|
500
|
+
source_url: msg.webLink,
|
|
501
|
+
occurred_at: new Date(msg.receivedDateTime),
|
|
502
|
+
origin_type: 'email',
|
|
503
|
+
metadata: {
|
|
504
|
+
from: msg.from?.emailAddress?.address,
|
|
505
|
+
to: formatRecipients(msg.toRecipients ?? []),
|
|
506
|
+
cc: formatRecipients(msg.ccRecipients ?? []),
|
|
507
|
+
importance: msg.importance,
|
|
508
|
+
has_attachments: msg.hasAttachments,
|
|
509
|
+
is_read: msg.isRead,
|
|
510
|
+
},
|
|
511
|
+
}),
|
|
512
|
+
// The next window starts where this one ended.
|
|
513
|
+
completed: { last_sync_at: window.end },
|
|
514
|
+
});
|
|
512
515
|
}
|
|
513
516
|
|
|
514
517
|
// -------------------------------------------------------------------------
|
|
@@ -517,68 +520,106 @@ export default class MicrosoftOutlookConnector extends ConnectorRuntime {
|
|
|
517
520
|
|
|
518
521
|
private async syncCalendar(ctx: SyncContext, http: HttpClient): Promise<SyncResult> {
|
|
519
522
|
const config = ctx.config as Record<string, unknown>;
|
|
520
|
-
const
|
|
521
|
-
const lookaheadDays = (config.lookahead_days as number) ?? 30;
|
|
522
|
-
const maxResults = (config.max_results as number) ?? 100;
|
|
523
|
+
const checkpoint = (ctx.checkpoint ?? {}) as OutlookCheckpoint;
|
|
523
524
|
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
const
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
525
|
+
// A calendar view is a snapshot of a moving range, re-walked in full on
|
|
526
|
+
// every traversal; a capped walk resumes the same range next run.
|
|
527
|
+
const window = checkpoint.pending?.window ?? {
|
|
528
|
+
start: daysFromNow(-((config.lookback_days as number) ?? 7)),
|
|
529
|
+
end: daysFromNow((config.lookahead_days as number) ?? 30),
|
|
530
|
+
};
|
|
530
531
|
const firstUrl =
|
|
531
532
|
`${this.API_BASE}/me/calendarView` +
|
|
532
|
-
`?startDateTime=${
|
|
533
|
-
`&endDateTime=${
|
|
534
|
-
`&$top=${
|
|
533
|
+
`?startDateTime=${window.start}` +
|
|
534
|
+
`&endDateTime=${window.end}` +
|
|
535
|
+
`&$top=${this.PAGE_SIZE}` +
|
|
535
536
|
'&$orderby=start/dateTime' +
|
|
536
537
|
'&$select=id,subject,bodyPreview,organizer,attendees,start,end,location,isAllDay,isCancelled,webLink,createdDateTime';
|
|
537
538
|
|
|
539
|
+
return this.walkPages<GraphEvent>(ctx, http, {
|
|
540
|
+
window,
|
|
541
|
+
firstUrl,
|
|
542
|
+
maxResults: (config.max_results as number) ?? 100,
|
|
543
|
+
toEvent: (evt) => ({
|
|
544
|
+
origin_id: `outlook_evt_${evt.id}`,
|
|
545
|
+
title: evt.subject,
|
|
546
|
+
payload_text: evt.bodyPreview || evt.subject,
|
|
547
|
+
author_name: evt.organizer?.emailAddress?.name || evt.organizer?.emailAddress?.address,
|
|
548
|
+
source_url: evt.webLink,
|
|
549
|
+
occurred_at: new Date(evt.start.dateTime),
|
|
550
|
+
origin_type: 'calendar_event',
|
|
551
|
+
metadata: {
|
|
552
|
+
organizer: evt.organizer?.emailAddress?.address,
|
|
553
|
+
location: evt.location?.displayName,
|
|
554
|
+
attendee_count: evt.attendees?.length ?? 0,
|
|
555
|
+
is_all_day: evt.isAllDay,
|
|
556
|
+
is_cancelled: evt.isCancelled,
|
|
557
|
+
start_time: evt.start.dateTime,
|
|
558
|
+
end_time: evt.end.dateTime,
|
|
559
|
+
},
|
|
560
|
+
}),
|
|
561
|
+
completed: { last_sync_at: new Date().toISOString() },
|
|
562
|
+
});
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/**
|
|
566
|
+
* Walk a Graph collection page by page, committing each page with the
|
|
567
|
+
* cursor of the next. `max_results` caps how much a run starts, never how
|
|
568
|
+
* much of a fetched page is stored: a page is committed whole, and the run
|
|
569
|
+
* stops between pages once the cap is reached, handing back `more`.
|
|
570
|
+
*/
|
|
571
|
+
private async walkPages<T>(
|
|
572
|
+
ctx: SyncContext,
|
|
573
|
+
http: HttpClient,
|
|
574
|
+
walk: {
|
|
575
|
+
window: { start: string; end: string };
|
|
576
|
+
firstUrl: string;
|
|
577
|
+
maxResults: number;
|
|
578
|
+
toEvent: (item: T) => EventEnvelope;
|
|
579
|
+
completed: OutlookCheckpoint;
|
|
580
|
+
}
|
|
581
|
+
): Promise<SyncResult> {
|
|
582
|
+
const checkpoint = (ctx.checkpoint ?? {}) as OutlookCheckpoint;
|
|
583
|
+
let url = checkpoint.pending?.next_link ?? walk.firstUrl;
|
|
584
|
+
// A saved link outlives the run that issued it and Graph may retire it.
|
|
585
|
+
// The first request of the run — the only one using a stored cursor — may
|
|
586
|
+
// restart the window from its first page instead of wedging the feed;
|
|
587
|
+
// `origin_id` makes that replay a supersede, not a duplicate.
|
|
588
|
+
let staleCursorRecoverable = checkpoint.pending !== undefined;
|
|
538
589
|
let fetched = 0;
|
|
539
590
|
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
}
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
origin_id: `outlook_evt_${evt.id}`,
|
|
553
|
-
title: evt.subject,
|
|
554
|
-
payload_text: evt.bodyPreview || evt.subject,
|
|
555
|
-
author_name: evt.organizer?.emailAddress?.name || evt.organizer?.emailAddress?.address,
|
|
556
|
-
source_url: evt.webLink,
|
|
557
|
-
occurred_at: new Date(evt.start.dateTime),
|
|
558
|
-
origin_type: 'calendar_event',
|
|
559
|
-
metadata: {
|
|
560
|
-
organizer: evt.organizer?.emailAddress?.address,
|
|
561
|
-
location: evt.location?.displayName,
|
|
562
|
-
attendee_count: evt.attendees?.length ?? 0,
|
|
563
|
-
is_all_day: evt.isAllDay,
|
|
564
|
-
is_cancelled: evt.isCancelled,
|
|
565
|
-
start_time: evt.start.dateTime,
|
|
566
|
-
end_time: evt.end.dateTime,
|
|
567
|
-
},
|
|
568
|
-
});
|
|
569
|
-
fetched++;
|
|
591
|
+
for (let page = 0; page < this.MAX_PAGES; page++) {
|
|
592
|
+
let data: GraphPagedResponse<T>;
|
|
593
|
+
try {
|
|
594
|
+
data = await http.get<GraphPagedResponse<T>>(url);
|
|
595
|
+
} catch (error) {
|
|
596
|
+
if (staleCursorRecoverable && error instanceof HttpStatusError && [400, 404, 410].includes(error.status)) {
|
|
597
|
+
staleCursorRecoverable = false;
|
|
598
|
+
url = walk.firstUrl;
|
|
599
|
+
page--;
|
|
600
|
+
continue;
|
|
601
|
+
}
|
|
602
|
+
throw error;
|
|
570
603
|
}
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
604
|
+
staleCursorRecoverable = false;
|
|
605
|
+
|
|
606
|
+
const events = data.value.map(walk.toEvent);
|
|
607
|
+
fetched += events.length;
|
|
608
|
+
const nextLink = data['@odata.nextLink'];
|
|
609
|
+
if (!nextLink) {
|
|
610
|
+
await ctx.commit(events, walk.completed as Record<string, unknown>);
|
|
611
|
+
return { status: 'complete', metadata: { items_found: fetched } };
|
|
612
|
+
}
|
|
613
|
+
const pending: OutlookCheckpoint = {
|
|
614
|
+
...(checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
|
|
615
|
+
pending: { window: walk.window, next_link: nextLink },
|
|
616
|
+
};
|
|
617
|
+
await ctx.commit(events, pending as Record<string, unknown>);
|
|
618
|
+
url = nextLink;
|
|
619
|
+
if (fetched >= walk.maxResults) break;
|
|
575
620
|
}
|
|
576
|
-
|
|
577
|
-
return {
|
|
578
|
-
events,
|
|
579
|
-
checkpoint: {
|
|
580
|
-
last_sync_at: new Date().toISOString(),
|
|
581
|
-
} satisfies OutlookCheckpoint as Record<string, unknown>,
|
|
582
|
-
};
|
|
621
|
+
// Pages remain: the committed cursor resumes this window next run.
|
|
622
|
+
return { status: 'more', metadata: { items_found: fetched } };
|
|
583
623
|
}
|
|
624
|
+
|
|
584
625
|
}
|
|
@@ -344,6 +344,9 @@ function toDate(v: unknown): Date | null {
|
|
|
344
344
|
return null;
|
|
345
345
|
}
|
|
346
346
|
|
|
347
|
+
/** Rows per `ctx.commit` during a sync. */
|
|
348
|
+
const COMMIT_ROWS = 500;
|
|
349
|
+
|
|
347
350
|
export default class PostgresConnector extends ConnectorRuntime {
|
|
348
351
|
readonly definition: RuntimeConnectorDefinition = {
|
|
349
352
|
key: 'postgres',
|
|
@@ -466,21 +469,26 @@ export default class PostgresConnector extends ConnectorRuntime {
|
|
|
466
469
|
// to feedKey only for direct/programmatic sync calls (no feedId).
|
|
467
470
|
const originPrefix = ctx.feedId != null ? String(ctx.feedId) : ctx.feedKey;
|
|
468
471
|
|
|
469
|
-
// Map rows → events
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
for (
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
472
|
+
// Map rows → events and commit them in slices, each with the keyset
|
|
473
|
+
// checkpoint of its last row: that checkpoint resumes exactly after it,
|
|
474
|
+
// so a run cut short keeps every slice it committed.
|
|
475
|
+
for (let start = 0; start < rows.length; start += COMMIT_ROWS) {
|
|
476
|
+
const slice = rows.slice(start, start + COMMIT_ROWS);
|
|
477
|
+
const last = slice[slice.length - 1];
|
|
478
|
+
const sliceCheckpoint: PgCheckpoint = {
|
|
479
|
+
last_cursor: toCheckpointValue(last[cursorCol]),
|
|
480
|
+
last_pk: toCheckpointValue(last[pkCol]),
|
|
477
481
|
};
|
|
482
|
+
await ctx.commit(
|
|
483
|
+
slice.map((row) => this.rowToEvent(originPrefix, row, config, cursorCol, pkCol)),
|
|
484
|
+
sliceCheckpoint as unknown as Record<string, unknown>
|
|
485
|
+
);
|
|
478
486
|
}
|
|
479
487
|
|
|
480
488
|
return {
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
metadata: { items_found:
|
|
489
|
+
// A full page means rows remain past the cursor: run again now.
|
|
490
|
+
status: rows.length >= limit ? 'more' : 'complete',
|
|
491
|
+
metadata: { items_found: rows.length },
|
|
484
492
|
};
|
|
485
493
|
} finally {
|
|
486
494
|
await sql.end({ timeout: 5 });
|
|
@@ -265,10 +265,12 @@ export default class ProductHuntConnector extends ConnectorRuntime {
|
|
|
265
265
|
|
|
266
266
|
let result: ProductHuntPostsResponse;
|
|
267
267
|
try {
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
268
|
+
// A GraphQL query sent as POST: safe to repeat on a 5xx.
|
|
269
|
+
result = await http.post<ProductHuntPostsResponse>(
|
|
270
|
+
this.API_URL,
|
|
271
|
+
{ query: LIST_POSTS_QUERY, variables },
|
|
272
|
+
{ idempotent: true }
|
|
273
|
+
);
|
|
272
274
|
} catch (error) {
|
|
273
275
|
if (error instanceof HttpStatusError && error.status === 401) {
|
|
274
276
|
// PH v2 API requires auth. Fail the run instead of returning an
|
|
@@ -352,9 +354,9 @@ export default class ProductHuntConnector extends ConnectorRuntime {
|
|
|
352
354
|
last_sync_at: new Date().toISOString(),
|
|
353
355
|
};
|
|
354
356
|
|
|
357
|
+
await ctx.commit(events, checkpoint as Record<string, unknown>);
|
|
355
358
|
return {
|
|
356
|
-
|
|
357
|
-
checkpoint: checkpoint as Record<string, unknown>,
|
|
359
|
+
status: 'complete',
|
|
358
360
|
metadata: {
|
|
359
361
|
items_found: events.length,
|
|
360
362
|
},
|
|
@@ -362,9 +362,9 @@ export default class RedditConnector extends ConnectorRuntime {
|
|
|
362
362
|
pagination_token: after ?? undefined,
|
|
363
363
|
};
|
|
364
364
|
|
|
365
|
+
await ctx.commit(events, checkpoint as Record<string, unknown>);
|
|
365
366
|
return {
|
|
366
|
-
|
|
367
|
-
checkpoint: checkpoint as Record<string, unknown>,
|
|
367
|
+
status: 'complete',
|
|
368
368
|
metadata: {
|
|
369
369
|
items_found: events.length,
|
|
370
370
|
},
|
|
@@ -181,9 +181,9 @@ export default class RSSConnector extends ConnectorRuntime {
|
|
|
181
181
|
last_published_at: latestPublishedAt,
|
|
182
182
|
};
|
|
183
183
|
|
|
184
|
+
await ctx.commit(events, newCheckpoint as unknown as Record<string, unknown>);
|
|
184
185
|
return {
|
|
185
|
-
|
|
186
|
-
checkpoint: newCheckpoint as unknown as Record<string, unknown>,
|
|
186
|
+
status: 'complete',
|
|
187
187
|
metadata: {
|
|
188
188
|
items_found: events.length,
|
|
189
189
|
feeds_fetched: feedUrls.length - fetchErrors.length,
|
|
@@ -143,6 +143,25 @@ export const slackChatUserIdentity: ChatUserIdentity = {
|
|
|
143
143
|
buildUserKey(teamId, platformUserId) {
|
|
144
144
|
return normalizeSlackUserId(teamId, platformUserId);
|
|
145
145
|
},
|
|
146
|
+
userKeyScope(teamId) {
|
|
147
|
+
// No usable team = REFUSE, never "search the whole namespace". Two
|
|
148
|
+
// workspaces can both hold `U12345`, so an unconstrained search here would
|
|
149
|
+
// hand back an id from the wrong one. Validate exactly as
|
|
150
|
+
// `normalizeSlackUserId` does so the prefix cannot disagree with the keys
|
|
151
|
+
// it wrote.
|
|
152
|
+
if (typeof teamId !== 'string') return null;
|
|
153
|
+
const t = teamId.trim();
|
|
154
|
+
if (!t || !/^[a-z0-9_-]+$/i.test(t)) return null;
|
|
155
|
+
return { kind: 'team-prefix', prefix: `${t.toUpperCase()}:` };
|
|
156
|
+
},
|
|
157
|
+
// Slack addresses a user by the BARE `U…`; the team prefix is storage scoping
|
|
158
|
+
// only and would not resolve if sent back. Re-validating through
|
|
159
|
+
// `normalizeTeamScopedId` first means a key this connector could not have
|
|
160
|
+
// written (no prefix, bad chars) yields null rather than a half-parsed id.
|
|
161
|
+
platformUserIdFromKey(key) {
|
|
162
|
+
const normalized = normalizeTeamScopedId(key);
|
|
163
|
+
return normalized ? normalized.slice(normalized.indexOf(':') + 1) : null;
|
|
164
|
+
},
|
|
146
165
|
};
|
|
147
166
|
|
|
148
167
|
/** A Slack channel and the (bare `U…`) ids of its members. */
|
|
@@ -937,7 +937,7 @@ export default class WhatsAppWebConnector extends ConnectorRuntime<
|
|
|
937
937
|
|
|
938
938
|
private async syncMessages(
|
|
939
939
|
ctx: SyncContext<BrowserCheckpoint, WhatsAppWebConfig>
|
|
940
|
-
): Promise<SyncResult
|
|
940
|
+
): Promise<SyncResult> {
|
|
941
941
|
const dispatcher = requireExtensionDispatcher(ctx);
|
|
942
942
|
const tabId = await readyWhatsAppTab(dispatcher);
|
|
943
943
|
return await this.collectMessages(ctx, dispatcher, tabId);
|
|
@@ -954,7 +954,7 @@ export default class WhatsAppWebConnector extends ConnectorRuntime<
|
|
|
954
954
|
ctx: SyncContext<BrowserCheckpoint, WhatsAppWebConfig>,
|
|
955
955
|
dispatcher: ChromeActionDispatcher,
|
|
956
956
|
tabId: number
|
|
957
|
-
): Promise<SyncResult
|
|
957
|
+
): Promise<SyncResult> {
|
|
958
958
|
try {
|
|
959
959
|
return await this.collectMessagesInner(ctx, dispatcher, tabId);
|
|
960
960
|
} catch (error) {
|
|
@@ -968,7 +968,7 @@ export default class WhatsAppWebConnector extends ConnectorRuntime<
|
|
|
968
968
|
ctx: SyncContext<BrowserCheckpoint, WhatsAppWebConfig>,
|
|
969
969
|
dispatcher: ChromeActionDispatcher,
|
|
970
970
|
tabId: number
|
|
971
|
-
): Promise<SyncResult
|
|
971
|
+
): Promise<SyncResult> {
|
|
972
972
|
const { checkpoint, request } = buildCollectionPlan({
|
|
973
973
|
checkpoint: ctx.checkpoint as Record<string, unknown> | null,
|
|
974
974
|
config: ctx.config as Record<string, unknown>,
|
|
@@ -1144,7 +1144,8 @@ export default class WhatsAppWebConnector extends ConnectorRuntime<
|
|
|
1144
1144
|
nextCheckpoint.diagnostics =
|
|
1145
1145
|
Object.keys(diagnostics).length > 0 ? diagnostics : undefined;
|
|
1146
1146
|
|
|
1147
|
-
|
|
1147
|
+
await ctx.commit(events, nextCheckpoint as unknown as BrowserCheckpoint);
|
|
1148
|
+
return { status: "complete" };
|
|
1148
1149
|
}
|
|
1149
1150
|
|
|
1150
1151
|
async execute(ctx: ActionContext): Promise<ActionResult> {
|
|
@@ -47,6 +47,13 @@ import {
|
|
|
47
47
|
type SyncContext,
|
|
48
48
|
type SyncResult,
|
|
49
49
|
} from "@lobu/connector-sdk";
|
|
50
|
+
|
|
51
|
+
/** What one X sync gathers before it is committed. */
|
|
52
|
+
type XSyncPage = {
|
|
53
|
+
events: EventEnvelope[];
|
|
54
|
+
checkpoint: Record<string, unknown>;
|
|
55
|
+
metadata?: SyncResult["metadata"];
|
|
56
|
+
};
|
|
50
57
|
import { X_IDENTITY, normalizeXHandle } from "./x-identity.js";
|
|
51
58
|
|
|
52
59
|
/** OAuth scopes needed per feed for the API path (not used to pause browser-capable feeds). */
|
|
@@ -1061,7 +1068,7 @@ export function finalizeDmSyncResult(
|
|
|
1061
1068
|
messages: XDmMessage[],
|
|
1062
1069
|
checkpoint: XCheckpoint,
|
|
1063
1070
|
metadata: Record<string, unknown>,
|
|
1064
|
-
):
|
|
1071
|
+
): XSyncPage {
|
|
1065
1072
|
const seenIds = new Set<string>();
|
|
1066
1073
|
const deduped = messages.filter((message) => {
|
|
1067
1074
|
if (!message.id || !message.text || seenIds.has(message.id)) return false;
|
|
@@ -1104,7 +1111,7 @@ export function finalizeSyncResult(
|
|
|
1104
1111
|
checkpoint: XCheckpoint,
|
|
1105
1112
|
metadata: Record<string, unknown>,
|
|
1106
1113
|
options?: { originType?: string },
|
|
1107
|
-
):
|
|
1114
|
+
): XSyncPage {
|
|
1108
1115
|
const seenIds = new Set<string>();
|
|
1109
1116
|
const deduped = tweets.filter((tweet) => {
|
|
1110
1117
|
if (!tweet.id || !tweet.text || seenIds.has(tweet.id)) return false;
|
|
@@ -1151,7 +1158,7 @@ export function finalizeLikedTweetsResult(
|
|
|
1151
1158
|
pagesRead: number;
|
|
1152
1159
|
historicalItemsRead: number;
|
|
1153
1160
|
},
|
|
1154
|
-
):
|
|
1161
|
+
): XSyncPage {
|
|
1155
1162
|
// A browser backfill intentionally re-observes the newest page on every run.
|
|
1156
1163
|
// Do not drop the old boundary: deterministic origin ids let ingestion update
|
|
1157
1164
|
// metrics/media without creating a second canonical event.
|
|
@@ -1498,7 +1505,7 @@ function shouldFallbackToExtension(error: unknown): boolean {
|
|
|
1498
1505
|
return isOAuthScopeOrAuthError(error) || isOAuthLookupFallbackError(error);
|
|
1499
1506
|
}
|
|
1500
1507
|
|
|
1501
|
-
async function syncWithOAuthFallback<T extends
|
|
1508
|
+
async function syncWithOAuthFallback<T extends XSyncPage>(
|
|
1502
1509
|
oauthFn: () => Promise<T>,
|
|
1503
1510
|
extensionFn: () => Promise<T>,
|
|
1504
1511
|
): Promise<T> {
|
|
@@ -1511,7 +1518,7 @@ async function syncWithOAuthFallback<T extends SyncResult>(
|
|
|
1511
1518
|
}
|
|
1512
1519
|
|
|
1513
1520
|
/** Browser-first feeds fall back to the extension unless `use_oauth` is set. */
|
|
1514
|
-
async function syncOAuthWithOptionalFallback<T extends
|
|
1521
|
+
async function syncOAuthWithOptionalFallback<T extends XSyncPage>(
|
|
1515
1522
|
config: Record<string, unknown>,
|
|
1516
1523
|
oauthFn: () => Promise<T>,
|
|
1517
1524
|
extensionFn: () => Promise<T>,
|
|
@@ -1774,7 +1781,7 @@ async function syncViaOAuthApi(
|
|
|
1774
1781
|
ctx: SyncContext,
|
|
1775
1782
|
config: Record<string, unknown>,
|
|
1776
1783
|
checkpoint: XCheckpoint,
|
|
1777
|
-
): Promise<
|
|
1784
|
+
): Promise<XSyncPage> {
|
|
1778
1785
|
const accessToken = ctx.credentials?.accessToken;
|
|
1779
1786
|
if (!accessToken) {
|
|
1780
1787
|
throw new Error("OAuth access token missing for X connector");
|
|
@@ -1876,7 +1883,7 @@ async function syncViaExtension(args: {
|
|
|
1876
1883
|
/** Extra metadata to fold into the result (e.g. which timeline tab). */
|
|
1877
1884
|
metadata?: Record<string, unknown>;
|
|
1878
1885
|
originType?: string;
|
|
1879
|
-
}): Promise<
|
|
1886
|
+
}): Promise<XSyncPage> {
|
|
1880
1887
|
const { ctx, url, interceptPatterns, parseResponse, maxScrolls, checkpoint } =
|
|
1881
1888
|
args;
|
|
1882
1889
|
const result = await extensionNetworkSync<XTweet>({
|
|
@@ -1909,7 +1916,7 @@ async function syncSearchViaExtension(
|
|
|
1909
1916
|
ctx: SyncContext,
|
|
1910
1917
|
config: Record<string, unknown>,
|
|
1911
1918
|
checkpoint: XCheckpoint,
|
|
1912
|
-
): Promise<
|
|
1919
|
+
): Promise<XSyncPage> {
|
|
1913
1920
|
const searchQuery = buildSearchQuery(config);
|
|
1914
1921
|
const maxScrolls = readMaxPages(config);
|
|
1915
1922
|
const searchFilter = (config.search_filter as string) ?? "live";
|
|
@@ -1930,7 +1937,7 @@ async function syncMyTweetsViaOAuthApi(
|
|
|
1930
1937
|
ctx: SyncContext,
|
|
1931
1938
|
config: Record<string, unknown>,
|
|
1932
1939
|
checkpoint: XCheckpoint,
|
|
1933
|
-
): Promise<
|
|
1940
|
+
): Promise<XSyncPage> {
|
|
1934
1941
|
const accessToken = ctx.credentials?.accessToken;
|
|
1935
1942
|
if (!accessToken) {
|
|
1936
1943
|
throw new Error("OAuth access token missing for my_tweets feed");
|
|
@@ -1977,7 +1984,7 @@ async function syncMyTweetsViaExtension(
|
|
|
1977
1984
|
ctx: SyncContext,
|
|
1978
1985
|
config: Record<string, unknown>,
|
|
1979
1986
|
checkpoint: XCheckpoint,
|
|
1980
|
-
): Promise<
|
|
1987
|
+
): Promise<XSyncPage> {
|
|
1981
1988
|
const accountHandle = await resolveAccountHandle(config);
|
|
1982
1989
|
const maxScrolls = readMaxPages(config);
|
|
1983
1990
|
const profileUrl = `https://x.com/${encodeURIComponent(accountHandle)}`;
|
|
@@ -1997,7 +2004,7 @@ async function syncLikedTweetsViaExtension(
|
|
|
1997
2004
|
ctx: SyncContext,
|
|
1998
2005
|
config: Record<string, unknown>,
|
|
1999
2006
|
checkpoint: XCheckpoint,
|
|
2000
|
-
): Promise<
|
|
2007
|
+
): Promise<XSyncPage> {
|
|
2001
2008
|
const accountHandle = await resolveAccountHandle(config);
|
|
2002
2009
|
const likesUrl = `https://x.com/${encodeURIComponent(accountHandle)}/likes`;
|
|
2003
2010
|
const dispatcher = requireExtensionDispatcher(ctx);
|
|
@@ -2130,7 +2137,7 @@ async function syncLikedTweetsViaOAuthApi(
|
|
|
2130
2137
|
ctx: SyncContext,
|
|
2131
2138
|
config: Record<string, unknown>,
|
|
2132
2139
|
checkpoint: XCheckpoint,
|
|
2133
|
-
): Promise<
|
|
2140
|
+
): Promise<XSyncPage> {
|
|
2134
2141
|
const accessToken = ctx.credentials?.accessToken;
|
|
2135
2142
|
if (!accessToken) {
|
|
2136
2143
|
throw new Error("OAuth access token missing for liked_tweets feed");
|
|
@@ -2201,7 +2208,7 @@ async function syncBookmarksViaOAuthApi(
|
|
|
2201
2208
|
ctx: SyncContext,
|
|
2202
2209
|
config: Record<string, unknown>,
|
|
2203
2210
|
checkpoint: XCheckpoint,
|
|
2204
|
-
): Promise<
|
|
2211
|
+
): Promise<XSyncPage> {
|
|
2205
2212
|
const accessToken = ctx.credentials?.accessToken;
|
|
2206
2213
|
if (!accessToken) {
|
|
2207
2214
|
throw new Error("OAuth access token missing for bookmarks feed");
|
|
@@ -2240,7 +2247,7 @@ async function syncBookmarksViaExtension(
|
|
|
2240
2247
|
ctx: SyncContext,
|
|
2241
2248
|
config: Record<string, unknown>,
|
|
2242
2249
|
checkpoint: XCheckpoint,
|
|
2243
|
-
): Promise<
|
|
2250
|
+
): Promise<XSyncPage> {
|
|
2244
2251
|
const maxScrolls = readMaxPages(config);
|
|
2245
2252
|
|
|
2246
2253
|
return syncViaExtension({
|
|
@@ -2278,7 +2285,7 @@ async function syncDirectMessagesViaOAuthApi(
|
|
|
2278
2285
|
ctx: SyncContext,
|
|
2279
2286
|
config: Record<string, unknown>,
|
|
2280
2287
|
checkpoint: XCheckpoint,
|
|
2281
|
-
): Promise<
|
|
2288
|
+
): Promise<XSyncPage> {
|
|
2282
2289
|
const accessToken = ctx.credentials?.accessToken;
|
|
2283
2290
|
if (!accessToken) {
|
|
2284
2291
|
throw new Error(
|
|
@@ -2332,7 +2339,7 @@ async function syncDirectMessagesViaExtension(
|
|
|
2332
2339
|
ctx: SyncContext,
|
|
2333
2340
|
config: Record<string, unknown>,
|
|
2334
2341
|
checkpoint: XCheckpoint,
|
|
2335
|
-
): Promise<
|
|
2342
|
+
): Promise<XSyncPage> {
|
|
2336
2343
|
const maxScrolls = readMaxPages(config);
|
|
2337
2344
|
let authUserId =
|
|
2338
2345
|
typeof config.account_user_id === "string"
|
|
@@ -2378,7 +2385,7 @@ async function syncHomeFeedViaDomScrape(
|
|
|
2378
2385
|
ctx: SyncContext,
|
|
2379
2386
|
config: Record<string, unknown>,
|
|
2380
2387
|
checkpoint: XCheckpoint,
|
|
2381
|
-
): Promise<
|
|
2388
|
+
): Promise<XSyncPage> {
|
|
2382
2389
|
const maxScrolls = readScrollBudget(config, { defaultMax: 10, cap: 30 });
|
|
2383
2390
|
const { items: rows, loggedIn } = await extensionDomScrape<HomeFeedRow>({
|
|
2384
2391
|
dispatcher: requireExtensionDispatcher(ctx),
|
|
@@ -3343,6 +3350,14 @@ export default class XConnector extends ConnectorRuntime {
|
|
|
3343
3350
|
}
|
|
3344
3351
|
|
|
3345
3352
|
private async syncFeed(ctx: SyncContext): Promise<SyncResult> {
|
|
3353
|
+
// Collected whole, then committed once: an OAuth attempt that fails part
|
|
3354
|
+
// way falls back to the extension, and nothing it gathered may land first.
|
|
3355
|
+
const page = await this.collectFeed(ctx);
|
|
3356
|
+
await ctx.commit(page.events, page.checkpoint);
|
|
3357
|
+
return { status: "complete", ...(page.metadata ? { metadata: page.metadata } : {}) };
|
|
3358
|
+
}
|
|
3359
|
+
|
|
3360
|
+
private async collectFeed(ctx: SyncContext): Promise<XSyncPage> {
|
|
3346
3361
|
const config = ctx.config as Record<string, unknown>;
|
|
3347
3362
|
const checkpoint = (ctx.checkpoint ?? {}) as XCheckpoint;
|
|
3348
3363
|
const feedKey = ctx.feedKey ?? "tweets";
|