@lobu/runtime-device 20.2.0 → 21.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/component.json +1 -1
- package/dist/dependencies.bun.lock +42 -36
- package/npm-shrinkwrap.json +270 -224
- package/package.json +1 -1
- package/vendor/cli/dist/commands/_lib/apply/apply-cmd.d.ts +7 -2
- package/vendor/cli/dist/commands/_lib/apply/apply-cmd.js +77 -75
- package/vendor/cli/dist/commands/_lib/apply/client.d.ts +40 -25
- package/vendor/cli/dist/commands/_lib/apply/client.js +31 -39
- package/vendor/cli/dist/commands/_lib/apply/deployment.d.ts +1 -1
- package/vendor/cli/dist/commands/_lib/apply/deployment.js +2 -2
- package/vendor/cli/dist/commands/_lib/apply/desired-state.d.ts +47 -11
- package/vendor/cli/dist/commands/_lib/apply/desired-state.js +106 -7
- package/vendor/cli/dist/commands/_lib/apply/diff.d.ts +14 -6
- package/vendor/cli/dist/commands/_lib/apply/diff.js +42 -23
- package/vendor/cli/dist/commands/_lib/apply/map-config.js +2 -3
- package/vendor/cli/dist/commands/_lib/apply/render.js +2 -0
- package/vendor/cli/dist/commands/_lib/apply/rollback-cmd.js +3 -3
- package/vendor/cli/dist/commands/_lib/connector-loader.d.ts +5 -1
- package/vendor/cli/dist/commands/_lib/connector-loader.js +1 -1
- package/vendor/cli/dist/commands/_lib/ensure-deps-installed.d.ts +15 -26
- package/vendor/cli/dist/commands/_lib/ensure-deps-installed.js +227 -136
- package/vendor/cli/dist/commands/_lib/view-bundler.d.ts +33 -0
- package/vendor/cli/dist/commands/_lib/view-bundler.js +207 -0
- package/vendor/cli/dist/commands/_lib/view-reapply.d.ts +60 -0
- package/vendor/cli/dist/commands/_lib/view-reapply.js +182 -0
- package/vendor/cli/dist/commands/dev.d.ts +14 -5
- package/vendor/cli/dist/commands/dev.js +56 -8
- package/vendor/cli/dist/commands/init.d.ts +1 -1
- package/vendor/cli/dist/commands/init.js +4 -4
- package/vendor/cli/dist/config/define.d.ts +25 -8
- package/vendor/cli/dist/config/define.js +24 -0
- package/vendor/cli/dist/connectors/README.md +15 -16
- package/vendor/cli/dist/connectors/github.ts +15 -9
- package/vendor/cli/dist/connectors/google_calendar.ts +21 -31
- package/vendor/cli/dist/connectors/google_drive.ts +106 -131
- package/vendor/cli/dist/connectors/google_gmail.ts +53 -30
- package/vendor/cli/dist/connectors/hackernews.ts +2 -5
- package/vendor/cli/dist/connectors/jira.ts +62 -27
- package/vendor/cli/dist/connectors/linear.ts +55 -21
- package/vendor/cli/dist/connectors/microsoft_outlook.ts +151 -110
- package/vendor/cli/dist/connectors/postgres.ts +19 -11
- package/vendor/cli/dist/connectors/producthunt.ts +8 -6
- package/vendor/cli/dist/connectors/reddit.ts +2 -2
- package/vendor/cli/dist/connectors/rss.ts +2 -2
- package/vendor/cli/dist/connectors/whatsapp_web.ts +5 -4
- package/vendor/cli/dist/connectors/x.ts +32 -17
- package/vendor/cli/dist/connectors/youtube.ts +18 -5
- package/vendor/cli/dist/internal/connector-compiler.d.ts +1 -1
- package/vendor/cli/dist/internal/connector-compiler.js +216 -17
- package/vendor/cli/dist/runtime-components.json +4 -4
- package/vendor/cli/package.json +2 -1
- package/vendor/connector-sdk/dist/connector-runtime.d.ts +4 -6
- package/vendor/connector-sdk/dist/connector-runtime.js +3 -5
- package/vendor/connector-sdk/dist/connector-types.d.ts +28 -13
- package/vendor/connector-sdk/dist/define-connector.d.ts +4 -1
- package/vendor/connector-sdk/dist/define-connector.js +4 -1
- package/vendor/connector-sdk/dist/http-client.d.ts +18 -7
- package/vendor/connector-sdk/dist/http-client.js +15 -13
- package/vendor/connector-sdk/dist/index.d.ts +1 -1
- package/vendor/connector-sdk/dist/retry.d.ts +15 -3
- package/vendor/connector-sdk/dist/retry.js +40 -64
- package/vendor/connector-sdk/package.json +1 -1
- package/vendor/connector-worker/dist/agent-turn/guest-entry.js +122 -10
- package/vendor/connector-worker/dist/agent-turn/guest.bundle.js +78 -10
- package/vendor/connector-worker/dist/agent-turn/types.d.ts +34 -4
- package/vendor/connector-worker/dist/compile/index.d.ts +21 -2
- package/vendor/connector-worker/dist/compile/index.js +41 -8
- package/vendor/connector-worker/dist/compile/source-capture.d.ts +10 -0
- package/vendor/connector-worker/dist/compile/source-capture.js +108 -0
- package/vendor/connector-worker/dist/daemon/agent-turn.js +97 -10
- package/vendor/connector-worker/dist/daemon/automation.d.ts +1 -1
- package/vendor/connector-worker/dist/daemon/executor.d.ts +0 -1
- package/vendor/connector-worker/dist/daemon/executor.js +42 -51
- package/vendor/connector-worker/dist/daemon/start.js +5 -1
- package/vendor/connector-worker/dist/executor/interface.d.ts +12 -8
- package/vendor/connector-worker/dist/executor/isolate.js +45 -28
- package/vendor/connector-worker/dist/self-check/index.js +15 -13
- package/vendor/connector-worker/package.json +1 -1
- package/vendor/core/dist/agent-policy.js +3 -0
- package/vendor/core/dist/connector-query-errors.d.ts +3 -3
- package/vendor/core/dist/connector-query-errors.js +8 -3
- package/vendor/core/dist/contracts/tools/manage-connections.d.ts +24 -0
- package/vendor/core/dist/contracts/tools/manage-connections.js +13 -0
- package/vendor/core/dist/contracts/tools/manage-entity-schema.d.ts +15 -25
- package/vendor/core/dist/contracts/tools/manage-entity-schema.js +3 -5
- package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +11 -0
- package/vendor/core/dist/contracts/tools/manage-feeds.js +12 -1
- package/vendor/core/dist/contracts/tools/manage-operations.d.ts +47 -6
- package/vendor/core/dist/contracts/tools/manage-operations.js +36 -1
- package/vendor/core/dist/contracts/tools/manage-views.d.ts +293 -0
- package/vendor/core/dist/contracts/tools/manage-views.js +203 -0
- package/vendor/core/dist/contracts/tools/source-files-schema.d.ts +7 -0
- package/vendor/core/dist/contracts/tools/source-files-schema.js +19 -0
- package/vendor/core/dist/contracts/tools/source-files.d.ts +14 -0
- package/vendor/core/dist/contracts/tools/source-files.js +55 -0
- package/vendor/core/dist/contracts/tools/view-content-hash.d.ts +37 -0
- package/vendor/core/dist/contracts/tools/view-content-hash.js +58 -0
- package/vendor/core/dist/contracts/tools/view-path.d.ts +18 -0
- package/vendor/core/dist/contracts/tools/view-path.js +24 -0
- package/vendor/core/dist/contracts/worker/protocol.d.ts +73 -8
- package/vendor/core/dist/contracts/worker/protocol.js +58 -9
- package/vendor/core/dist/reserved.d.ts +3 -3
- package/vendor/core/dist/reserved.js +9 -1
- package/vendor/core/dist/utils/secret-redaction.js +2 -0
- package/vendor/core/package.json +25 -5
- package/vendor/core/dist/contracts/tools/manage-view-templates.d.ts +0 -165
- package/vendor/core/dist/contracts/tools/manage-view-templates.js +0 -130
|
@@ -12,7 +12,7 @@ import {
|
|
|
12
12
|
type EventEnvelope,
|
|
13
13
|
type FeedReadContext,
|
|
14
14
|
type FeedReadResult,
|
|
15
|
-
|
|
15
|
+
HttpStatusError,
|
|
16
16
|
requireBearerClient,
|
|
17
17
|
type RuntimeConnectorDefinition,
|
|
18
18
|
type SyncContext,
|
|
@@ -64,7 +64,15 @@ interface JiraConfig {
|
|
|
64
64
|
}
|
|
65
65
|
|
|
66
66
|
interface JiraCheckpoint {
|
|
67
|
+
/** When the last complete traversal began; the next one fetches issues updated since. */
|
|
67
68
|
last_sync_at?: string;
|
|
69
|
+
/**
|
|
70
|
+
* A traversal that stopped at the per-run page cap: the fixed `updated`
|
|
71
|
+
* window it walks and the `nextPageToken` of the first page not yet stored,
|
|
72
|
+
* so the next run continues instead of re-reading the newest issues and
|
|
73
|
+
* never reaching the rest.
|
|
74
|
+
*/
|
|
75
|
+
pending?: { window: { start: string; end: string }; page_token: string };
|
|
68
76
|
}
|
|
69
77
|
|
|
70
78
|
interface JiraUser {
|
|
@@ -519,52 +527,79 @@ export default class JiraConnector extends ConnectorRuntime<JiraCheckpoint, Jira
|
|
|
519
527
|
// Sync into local memory.
|
|
520
528
|
// -------------------------------------------------------------------------
|
|
521
529
|
|
|
522
|
-
private async syncFeed(ctx: SyncContext<JiraCheckpoint, JiraConfig>): Promise<SyncResult
|
|
530
|
+
private async syncFeed(ctx: SyncContext<JiraCheckpoint, JiraConfig>): Promise<SyncResult> {
|
|
523
531
|
const base = await this.restBase(ctx.config, ctx.sessionState, ctx.credentials);
|
|
524
532
|
const http = this.client(ctx.credentials);
|
|
533
|
+
const checkpoint = ctx.checkpoint ?? {};
|
|
525
534
|
const lookbackDays = ctx.config.lookback_days ?? 365;
|
|
535
|
+
// Fixed per traversal: issues updated since the last complete traversal
|
|
536
|
+
// began, up to when this one began. An issue edited while the walk runs
|
|
537
|
+
// falls in the next window, and `origin_id` makes a re-report a supersede.
|
|
538
|
+
const window = checkpoint.pending?.window ?? {
|
|
539
|
+
start: checkpoint.last_sync_at ?? new Date(0).toISOString(),
|
|
540
|
+
end: new Date().toISOString(),
|
|
541
|
+
};
|
|
526
542
|
const jql = buildJiraJql({
|
|
527
543
|
baseQuery: asString(ctx.config.query) ?? asString(ctx.config.jql) ?? '',
|
|
528
544
|
defaultWhenEmpty: `updated >= -${lookbackDays}d`,
|
|
545
|
+
window,
|
|
529
546
|
});
|
|
530
547
|
|
|
531
|
-
const events: EventEnvelope[] = [];
|
|
532
|
-
|
|
533
548
|
// `/rest/api/3/search` was removed by Atlassian (CHANGE-2046); the
|
|
534
549
|
// replacement `/search/jql` paginates with an opaque nextPageToken and
|
|
535
550
|
// returns no total — iterate until the token is absent.
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
551
|
+
let pageToken = checkpoint.pending?.page_token;
|
|
552
|
+
// A stored token may be retired between runs. The first request of the run
|
|
553
|
+
// — the only one using a token from a previous run — may restart the same
|
|
554
|
+
// window from its first page instead of wedging the feed.
|
|
555
|
+
let staleCursorRecoverable = pageToken !== undefined;
|
|
556
|
+
let found = 0;
|
|
557
|
+
for (let page = 0; page < this.MAX_PAGES; page++) {
|
|
558
|
+
const params = new URLSearchParams({
|
|
559
|
+
jql,
|
|
560
|
+
maxResults: String(this.PAGE_SIZE),
|
|
561
|
+
fields: ISSUE_FIELDS,
|
|
562
|
+
});
|
|
563
|
+
if (pageToken) params.set('nextPageToken', pageToken);
|
|
564
|
+
let data: JiraSearchResponse;
|
|
565
|
+
try {
|
|
566
|
+
data = await http.json<JiraSearchResponse>(
|
|
545
567
|
`${base}/search/jql?${params.toString()}`,
|
|
546
568
|
{ method: 'GET', headers: { Accept: 'application/json' } },
|
|
547
569
|
);
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
570
|
+
} catch (error) {
|
|
571
|
+
if (staleCursorRecoverable && error instanceof HttpStatusError && error.status === 400) {
|
|
572
|
+
staleCursorRecoverable = false;
|
|
573
|
+
pageToken = undefined;
|
|
574
|
+
page--;
|
|
575
|
+
continue;
|
|
576
|
+
}
|
|
577
|
+
throw error;
|
|
578
|
+
}
|
|
579
|
+
staleCursorRecoverable = false;
|
|
552
580
|
|
|
553
|
-
|
|
581
|
+
const issues = data.issues ?? [];
|
|
582
|
+
const events: EventEnvelope[] = [];
|
|
554
583
|
for (const issue of issues) {
|
|
555
584
|
const event = this.issueEvent(issue, ctx.config);
|
|
556
585
|
if (event) events.push(event);
|
|
557
586
|
}
|
|
558
|
-
|
|
559
|
-
// returned — guards against a degenerate self-referential cursor.
|
|
560
|
-
if (issues.length === 0) break;
|
|
561
|
-
}
|
|
587
|
+
found += events.length;
|
|
562
588
|
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
589
|
+
// An empty page ends the walk even when a token is returned — guards
|
|
590
|
+
// against a degenerate self-referential cursor.
|
|
591
|
+
if (!data.nextPageToken || issues.length === 0) {
|
|
592
|
+
await ctx.commit(events, { last_sync_at: window.end });
|
|
593
|
+
return { status: 'complete', metadata: { items_found: found } };
|
|
594
|
+
}
|
|
595
|
+
pageToken = data.nextPageToken;
|
|
596
|
+
await ctx.commit(events, {
|
|
597
|
+
...(checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
|
|
598
|
+
pending: { window, page_token: pageToken },
|
|
599
|
+
});
|
|
600
|
+
}
|
|
601
|
+
// The page cap stopped the walk; its committed token resumes it next run.
|
|
602
|
+
return { status: 'more', metadata: { items_found: found } };
|
|
568
603
|
}
|
|
569
604
|
|
|
570
605
|
// -------------------------------------------------------------------------
|
|
@@ -40,7 +40,15 @@ interface LinearConfig {
|
|
|
40
40
|
}
|
|
41
41
|
|
|
42
42
|
interface LinearCheckpoint {
|
|
43
|
+
/** When the last complete traversal began; the next one fetches issues updated since. */
|
|
43
44
|
last_sync_at?: string;
|
|
45
|
+
/**
|
|
46
|
+
* A traversal that stopped at the per-run page cap. `since`/`started_at`
|
|
47
|
+
* are the fixed bounds it walks and `cursor` the `endCursor` of the last page
|
|
48
|
+
* stored, so the next run continues past it instead of re-reading the newest
|
|
49
|
+
* issues and never reaching the rest.
|
|
50
|
+
*/
|
|
51
|
+
pending?: { since?: string; started_at: string; cursor: string };
|
|
44
52
|
}
|
|
45
53
|
|
|
46
54
|
interface LinearUser {
|
|
@@ -297,7 +305,7 @@ export default class LinearConnector extends ConnectorRuntime<LinearCheckpoint,
|
|
|
297
305
|
pageInfo?: { hasNextPage?: boolean; endCursor?: string | null };
|
|
298
306
|
nodes?: LinearIssueNode[];
|
|
299
307
|
};
|
|
300
|
-
} = await this.graphql(ctx.credentials, gql);
|
|
308
|
+
} = await this.graphql(ctx.credentials, gql, undefined, { idempotent: true });
|
|
301
309
|
|
|
302
310
|
const rows = (response.issues?.nodes ?? [])
|
|
303
311
|
.map((node) => this.issueRow(node))
|
|
@@ -323,16 +331,24 @@ export default class LinearConnector extends ConnectorRuntime<LinearCheckpoint,
|
|
|
323
331
|
// Sync into local memory.
|
|
324
332
|
// -------------------------------------------------------------------------
|
|
325
333
|
|
|
326
|
-
private async syncFeed(ctx: SyncContext<LinearCheckpoint, LinearConfig>): Promise<SyncResult
|
|
327
|
-
const
|
|
328
|
-
|
|
329
|
-
|
|
334
|
+
private async syncFeed(ctx: SyncContext<LinearCheckpoint, LinearConfig>): Promise<SyncResult> {
|
|
335
|
+
const checkpoint = ctx.checkpoint ?? {};
|
|
336
|
+
// Fixed per traversal: issues updated after the last complete traversal
|
|
337
|
+
// began (none on the first), walked until the listing ends. An issue edited
|
|
338
|
+
// while the walk runs is re-reported by the next traversal, and `origin_id`
|
|
339
|
+
// makes that a supersede, not a duplicate.
|
|
340
|
+
const since = checkpoint.pending ? checkpoint.pending.since : checkpoint.last_sync_at;
|
|
341
|
+
const startedAt = checkpoint.pending?.started_at ?? new Date().toISOString();
|
|
342
|
+
let cursor: string | null = checkpoint.pending?.cursor ?? null;
|
|
330
343
|
|
|
331
|
-
const filter =
|
|
332
|
-
|
|
333
|
-
:
|
|
344
|
+
const filter = buildLinearIssueFilter({
|
|
345
|
+
teamKey: asString(ctx.config.team_key),
|
|
346
|
+
updatedAfterIso: since,
|
|
347
|
+
updatedBeforeIso: startedAt,
|
|
348
|
+
});
|
|
334
349
|
|
|
335
|
-
|
|
350
|
+
let found = 0;
|
|
351
|
+
for (let pages = 0; pages < this.MAX_PAGES; pages++) {
|
|
336
352
|
const after: string = cursor ? `, after: ${JSON.stringify(cursor)}` : '';
|
|
337
353
|
const query: string = `
|
|
338
354
|
query {
|
|
@@ -348,25 +364,36 @@ export default class LinearConnector extends ConnectorRuntime<LinearCheckpoint,
|
|
|
348
364
|
pageInfo?: { hasNextPage?: boolean; endCursor?: string | null };
|
|
349
365
|
nodes?: LinearIssueNode[];
|
|
350
366
|
};
|
|
351
|
-
}>(ctx.credentials, query);
|
|
367
|
+
}>(ctx.credentials, query, undefined, { idempotent: true });
|
|
352
368
|
|
|
353
|
-
const nodes = response.issues?.nodes
|
|
369
|
+
const nodes = response.issues?.nodes;
|
|
370
|
+
const pageInfo = response.issues?.pageInfo;
|
|
371
|
+
if (
|
|
372
|
+
!Array.isArray(nodes) ||
|
|
373
|
+
typeof pageInfo?.hasNextPage !== 'boolean' ||
|
|
374
|
+
(pageInfo.hasNextPage && !pageInfo.endCursor)
|
|
375
|
+
) {
|
|
376
|
+
throw new Error('Linear did not return a valid page cursor/exhaustion state.');
|
|
377
|
+
}
|
|
378
|
+
const events: EventEnvelope[] = [];
|
|
354
379
|
for (const node of nodes) {
|
|
355
380
|
const event = this.issueEvent(node);
|
|
356
381
|
if (event) events.push(event);
|
|
357
382
|
}
|
|
383
|
+
found += events.length;
|
|
358
384
|
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
385
|
+
if (!pageInfo?.hasNextPage || !pageInfo.endCursor) {
|
|
386
|
+
await ctx.commit(events, { last_sync_at: startedAt });
|
|
387
|
+
return { status: 'complete', metadata: { items_found: found } };
|
|
388
|
+
}
|
|
362
389
|
cursor = pageInfo.endCursor;
|
|
390
|
+
await ctx.commit(events, {
|
|
391
|
+
...(checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
|
|
392
|
+
pending: { ...(since ? { since } : {}), started_at: startedAt, cursor },
|
|
393
|
+
});
|
|
363
394
|
}
|
|
364
|
-
|
|
365
|
-
return {
|
|
366
|
-
events,
|
|
367
|
-
checkpoint: { last_sync_at: new Date().toISOString() },
|
|
368
|
-
metadata: { items_found: events.length },
|
|
369
|
-
};
|
|
395
|
+
// The page cap stopped the walk; its committed cursor resumes it next run.
|
|
396
|
+
return { status: 'more', metadata: { items_found: found } };
|
|
370
397
|
}
|
|
371
398
|
|
|
372
399
|
// -------------------------------------------------------------------------
|
|
@@ -463,10 +490,16 @@ export default class LinearConnector extends ConnectorRuntime<LinearCheckpoint,
|
|
|
463
490
|
// GraphQL transport
|
|
464
491
|
// -------------------------------------------------------------------------
|
|
465
492
|
|
|
493
|
+
/**
|
|
494
|
+
* `idempotent` marks a read: a GraphQL query goes out as POST, so without it
|
|
495
|
+
* the client never repeats the request on a 5xx. Leave it off for mutations —
|
|
496
|
+
* a 5xx does not say whether the webhook was already created.
|
|
497
|
+
*/
|
|
466
498
|
private async graphql<T>(
|
|
467
499
|
credentials: SyncCredentials | null,
|
|
468
500
|
query: string,
|
|
469
|
-
variables?: Record<string, unknown
|
|
501
|
+
variables?: Record<string, unknown>,
|
|
502
|
+
options: { idempotent?: boolean } = {}
|
|
470
503
|
): Promise<T> {
|
|
471
504
|
const http = requireBearerClient(credentials, {
|
|
472
505
|
errorPrefix: 'Linear API',
|
|
@@ -478,6 +511,7 @@ export default class LinearConnector extends ConnectorRuntime<LinearCheckpoint,
|
|
|
478
511
|
method: 'POST',
|
|
479
512
|
headers: { 'Content-Type': 'application/json' },
|
|
480
513
|
body: JSON.stringify({ query, variables: variables ?? {} }),
|
|
514
|
+
idempotent: options.idempotent,
|
|
481
515
|
}
|
|
482
516
|
);
|
|
483
517
|
|
|
@@ -14,9 +14,9 @@ import {
|
|
|
14
14
|
type EventEnvelope,
|
|
15
15
|
fileDownloadOutput,
|
|
16
16
|
type HttpClient,
|
|
17
|
+
HttpStatusError,
|
|
17
18
|
inlineContentBudget,
|
|
18
19
|
inlineMaxBytesSchema,
|
|
19
|
-
paginateByCursor,
|
|
20
20
|
requireBearerClient,
|
|
21
21
|
type SyncContext,
|
|
22
22
|
type SyncResult,
|
|
@@ -89,7 +89,24 @@ interface GraphPagedResponse<T> {
|
|
|
89
89
|
// ---------------------------------------------------------------------------
|
|
90
90
|
|
|
91
91
|
interface OutlookCheckpoint {
|
|
92
|
+
/** When the last complete traversal began; the next window starts here. */
|
|
92
93
|
last_sync_at?: string;
|
|
94
|
+
/**
|
|
95
|
+
* A traversal that stopped at the per-run cap with pages left. `window`
|
|
96
|
+
* is the fixed range it walks and `next_link` Graph's cursor for the first
|
|
97
|
+
* page not yet stored, so the next run continues exactly where this one
|
|
98
|
+
* stopped instead of re-reading the newest page and never reaching the rest.
|
|
99
|
+
*/
|
|
100
|
+
pending?: { window: { start: string; end: string }; next_link: string };
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
const WINDOW_OVERLAP_MS = 5 * 60_000;
|
|
104
|
+
|
|
105
|
+
/** ISO timestamp `days` from now (negative for the past). */
|
|
106
|
+
function daysFromNow(days: number): string {
|
|
107
|
+
const date = new Date();
|
|
108
|
+
date.setDate(date.getDate() + days);
|
|
109
|
+
return date.toISOString();
|
|
93
110
|
}
|
|
94
111
|
|
|
95
112
|
// ---------------------------------------------------------------------------
|
|
@@ -448,67 +465,53 @@ export default class MicrosoftOutlookConnector extends ConnectorRuntime {
|
|
|
448
465
|
private async syncMessages(ctx: SyncContext, http: HttpClient): Promise<SyncResult> {
|
|
449
466
|
const config = ctx.config as Record<string, unknown>;
|
|
450
467
|
const folder = (config.folder as string) ?? 'inbox';
|
|
451
|
-
const maxResults = (config.max_results as number) ?? 50;
|
|
452
468
|
const lookbackDays = (config.lookback_days as number) ?? 30;
|
|
453
|
-
const
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
469
|
+
const checkpoint = (ctx.checkpoint ?? {}) as OutlookCheckpoint;
|
|
470
|
+
|
|
471
|
+
// A fixed window per traversal: from where the last complete one began
|
|
472
|
+
// (or the lookback) up to when this one began. Mail arriving meanwhile
|
|
473
|
+
// belongs to the next window, so a capped walk can never be pushed past
|
|
474
|
+
// messages it has not reached by newer arrivals landing at the front.
|
|
475
|
+
// The start overlaps the previous window by a few minutes: Graph can
|
|
476
|
+
// surface a message after its receivedDateTime, and `origin_id` makes the
|
|
477
|
+
// replayed boundary a supersede, not a duplicate.
|
|
478
|
+
const window = checkpoint.pending?.window ?? {
|
|
479
|
+
start: checkpoint.last_sync_at
|
|
480
|
+
? new Date(Date.parse(checkpoint.last_sync_at) - WINDOW_OVERLAP_MS).toISOString()
|
|
481
|
+
: daysFromNow(-lookbackDays),
|
|
482
|
+
end: new Date().toISOString(),
|
|
483
|
+
};
|
|
460
484
|
const firstUrl =
|
|
461
|
-
`${this.API_BASE}/me/mailFolders/${
|
|
462
|
-
`?$top=${
|
|
485
|
+
`${this.API_BASE}/me/mailFolders/${encodeURIComponent(folder)}/messages` +
|
|
486
|
+
`?$top=${this.PAGE_SIZE}` +
|
|
463
487
|
'&$orderby=receivedDateTime desc' +
|
|
464
|
-
`&$filter=receivedDateTime ge ${
|
|
488
|
+
`&$filter=receivedDateTime ge ${window.start} and receivedDateTime lt ${window.end}` +
|
|
465
489
|
'&$select=id,conversationId,subject,bodyPreview,from,toRecipients,ccRecipients,receivedDateTime,sentDateTime,hasAttachments,importance,isRead,webLink';
|
|
466
490
|
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
|
|
490
|
-
|
|
491
|
-
to: formatRecipients(msg.toRecipients ?? []),
|
|
492
|
-
cc: formatRecipients(msg.ccRecipients ?? []),
|
|
493
|
-
importance: msg.importance,
|
|
494
|
-
has_attachments: msg.hasAttachments,
|
|
495
|
-
is_read: msg.isRead,
|
|
496
|
-
},
|
|
497
|
-
});
|
|
498
|
-
fetched++;
|
|
499
|
-
}
|
|
500
|
-
|
|
501
|
-
if (ctx.emitEvents) await ctx.emitEvents(events.splice(0));
|
|
502
|
-
|
|
503
|
-
if (fetched >= maxResults) break;
|
|
504
|
-
}
|
|
505
|
-
|
|
506
|
-
return {
|
|
507
|
-
events,
|
|
508
|
-
checkpoint: {
|
|
509
|
-
last_sync_at: new Date().toISOString(),
|
|
510
|
-
} satisfies OutlookCheckpoint as Record<string, unknown>,
|
|
511
|
-
};
|
|
491
|
+
return this.walkPages<GraphMessage>(ctx, http, {
|
|
492
|
+
window,
|
|
493
|
+
firstUrl,
|
|
494
|
+
maxResults: (config.max_results as number) ?? 50,
|
|
495
|
+
toEvent: (msg) => ({
|
|
496
|
+
origin_id: `outlook_msg_${msg.id}`,
|
|
497
|
+
title: msg.subject,
|
|
498
|
+
payload_text: msg.bodyPreview || msg.subject,
|
|
499
|
+
author_name: msg.from?.emailAddress?.name || msg.from?.emailAddress?.address,
|
|
500
|
+
source_url: msg.webLink,
|
|
501
|
+
occurred_at: new Date(msg.receivedDateTime),
|
|
502
|
+
origin_type: 'email',
|
|
503
|
+
metadata: {
|
|
504
|
+
from: msg.from?.emailAddress?.address,
|
|
505
|
+
to: formatRecipients(msg.toRecipients ?? []),
|
|
506
|
+
cc: formatRecipients(msg.ccRecipients ?? []),
|
|
507
|
+
importance: msg.importance,
|
|
508
|
+
has_attachments: msg.hasAttachments,
|
|
509
|
+
is_read: msg.isRead,
|
|
510
|
+
},
|
|
511
|
+
}),
|
|
512
|
+
// The next window starts where this one ended.
|
|
513
|
+
completed: { last_sync_at: window.end },
|
|
514
|
+
});
|
|
512
515
|
}
|
|
513
516
|
|
|
514
517
|
// -------------------------------------------------------------------------
|
|
@@ -517,68 +520,106 @@ export default class MicrosoftOutlookConnector extends ConnectorRuntime {
|
|
|
517
520
|
|
|
518
521
|
private async syncCalendar(ctx: SyncContext, http: HttpClient): Promise<SyncResult> {
|
|
519
522
|
const config = ctx.config as Record<string, unknown>;
|
|
520
|
-
const
|
|
521
|
-
const lookaheadDays = (config.lookahead_days as number) ?? 30;
|
|
522
|
-
const maxResults = (config.max_results as number) ?? 100;
|
|
523
|
+
const checkpoint = (ctx.checkpoint ?? {}) as OutlookCheckpoint;
|
|
523
524
|
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
const
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
525
|
+
// A calendar view is a snapshot of a moving range, re-walked in full on
|
|
526
|
+
// every traversal; a capped walk resumes the same range next run.
|
|
527
|
+
const window = checkpoint.pending?.window ?? {
|
|
528
|
+
start: daysFromNow(-((config.lookback_days as number) ?? 7)),
|
|
529
|
+
end: daysFromNow((config.lookahead_days as number) ?? 30),
|
|
530
|
+
};
|
|
530
531
|
const firstUrl =
|
|
531
532
|
`${this.API_BASE}/me/calendarView` +
|
|
532
|
-
`?startDateTime=${
|
|
533
|
-
`&endDateTime=${
|
|
534
|
-
`&$top=${
|
|
533
|
+
`?startDateTime=${window.start}` +
|
|
534
|
+
`&endDateTime=${window.end}` +
|
|
535
|
+
`&$top=${this.PAGE_SIZE}` +
|
|
535
536
|
'&$orderby=start/dateTime' +
|
|
536
537
|
'&$select=id,subject,bodyPreview,organizer,attendees,start,end,location,isAllDay,isCancelled,webLink,createdDateTime';
|
|
537
538
|
|
|
539
|
+
return this.walkPages<GraphEvent>(ctx, http, {
|
|
540
|
+
window,
|
|
541
|
+
firstUrl,
|
|
542
|
+
maxResults: (config.max_results as number) ?? 100,
|
|
543
|
+
toEvent: (evt) => ({
|
|
544
|
+
origin_id: `outlook_evt_${evt.id}`,
|
|
545
|
+
title: evt.subject,
|
|
546
|
+
payload_text: evt.bodyPreview || evt.subject,
|
|
547
|
+
author_name: evt.organizer?.emailAddress?.name || evt.organizer?.emailAddress?.address,
|
|
548
|
+
source_url: evt.webLink,
|
|
549
|
+
occurred_at: new Date(evt.start.dateTime),
|
|
550
|
+
origin_type: 'calendar_event',
|
|
551
|
+
metadata: {
|
|
552
|
+
organizer: evt.organizer?.emailAddress?.address,
|
|
553
|
+
location: evt.location?.displayName,
|
|
554
|
+
attendee_count: evt.attendees?.length ?? 0,
|
|
555
|
+
is_all_day: evt.isAllDay,
|
|
556
|
+
is_cancelled: evt.isCancelled,
|
|
557
|
+
start_time: evt.start.dateTime,
|
|
558
|
+
end_time: evt.end.dateTime,
|
|
559
|
+
},
|
|
560
|
+
}),
|
|
561
|
+
completed: { last_sync_at: new Date().toISOString() },
|
|
562
|
+
});
|
|
563
|
+
}
|
|
564
|
+
|
|
565
|
+
/**
|
|
566
|
+
* Walk a Graph collection page by page, committing each page with the
|
|
567
|
+
* cursor of the next. `max_results` caps how much a run starts, never how
|
|
568
|
+
* much of a fetched page is stored: a page is committed whole, and the run
|
|
569
|
+
* stops between pages once the cap is reached, handing back `more`.
|
|
570
|
+
*/
|
|
571
|
+
private async walkPages<T>(
|
|
572
|
+
ctx: SyncContext,
|
|
573
|
+
http: HttpClient,
|
|
574
|
+
walk: {
|
|
575
|
+
window: { start: string; end: string };
|
|
576
|
+
firstUrl: string;
|
|
577
|
+
maxResults: number;
|
|
578
|
+
toEvent: (item: T) => EventEnvelope;
|
|
579
|
+
completed: OutlookCheckpoint;
|
|
580
|
+
}
|
|
581
|
+
): Promise<SyncResult> {
|
|
582
|
+
const checkpoint = (ctx.checkpoint ?? {}) as OutlookCheckpoint;
|
|
583
|
+
let url = checkpoint.pending?.next_link ?? walk.firstUrl;
|
|
584
|
+
// A saved link outlives the run that issued it and Graph may retire it.
|
|
585
|
+
// The first request of the run — the only one using a stored cursor — may
|
|
586
|
+
// restart the window from its first page instead of wedging the feed;
|
|
587
|
+
// `origin_id` makes that replay a supersede, not a duplicate.
|
|
588
|
+
let staleCursorRecoverable = checkpoint.pending !== undefined;
|
|
538
589
|
let fetched = 0;
|
|
539
590
|
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
}
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
origin_id: `outlook_evt_${evt.id}`,
|
|
553
|
-
title: evt.subject,
|
|
554
|
-
payload_text: evt.bodyPreview || evt.subject,
|
|
555
|
-
author_name: evt.organizer?.emailAddress?.name || evt.organizer?.emailAddress?.address,
|
|
556
|
-
source_url: evt.webLink,
|
|
557
|
-
occurred_at: new Date(evt.start.dateTime),
|
|
558
|
-
origin_type: 'calendar_event',
|
|
559
|
-
metadata: {
|
|
560
|
-
organizer: evt.organizer?.emailAddress?.address,
|
|
561
|
-
location: evt.location?.displayName,
|
|
562
|
-
attendee_count: evt.attendees?.length ?? 0,
|
|
563
|
-
is_all_day: evt.isAllDay,
|
|
564
|
-
is_cancelled: evt.isCancelled,
|
|
565
|
-
start_time: evt.start.dateTime,
|
|
566
|
-
end_time: evt.end.dateTime,
|
|
567
|
-
},
|
|
568
|
-
});
|
|
569
|
-
fetched++;
|
|
591
|
+
for (let page = 0; page < this.MAX_PAGES; page++) {
|
|
592
|
+
let data: GraphPagedResponse<T>;
|
|
593
|
+
try {
|
|
594
|
+
data = await http.get<GraphPagedResponse<T>>(url);
|
|
595
|
+
} catch (error) {
|
|
596
|
+
if (staleCursorRecoverable && error instanceof HttpStatusError && [400, 404, 410].includes(error.status)) {
|
|
597
|
+
staleCursorRecoverable = false;
|
|
598
|
+
url = walk.firstUrl;
|
|
599
|
+
page--;
|
|
600
|
+
continue;
|
|
601
|
+
}
|
|
602
|
+
throw error;
|
|
570
603
|
}
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
604
|
+
staleCursorRecoverable = false;
|
|
605
|
+
|
|
606
|
+
const events = data.value.map(walk.toEvent);
|
|
607
|
+
fetched += events.length;
|
|
608
|
+
const nextLink = data['@odata.nextLink'];
|
|
609
|
+
if (!nextLink) {
|
|
610
|
+
await ctx.commit(events, walk.completed as Record<string, unknown>);
|
|
611
|
+
return { status: 'complete', metadata: { items_found: fetched } };
|
|
612
|
+
}
|
|
613
|
+
const pending: OutlookCheckpoint = {
|
|
614
|
+
...(checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
|
|
615
|
+
pending: { window: walk.window, next_link: nextLink },
|
|
616
|
+
};
|
|
617
|
+
await ctx.commit(events, pending as Record<string, unknown>);
|
|
618
|
+
url = nextLink;
|
|
619
|
+
if (fetched >= walk.maxResults) break;
|
|
575
620
|
}
|
|
576
|
-
|
|
577
|
-
return {
|
|
578
|
-
events,
|
|
579
|
-
checkpoint: {
|
|
580
|
-
last_sync_at: new Date().toISOString(),
|
|
581
|
-
} satisfies OutlookCheckpoint as Record<string, unknown>,
|
|
582
|
-
};
|
|
621
|
+
// Pages remain: the committed cursor resumes this window next run.
|
|
622
|
+
return { status: 'more', metadata: { items_found: fetched } };
|
|
583
623
|
}
|
|
624
|
+
|
|
584
625
|
}
|
|
@@ -344,6 +344,9 @@ function toDate(v: unknown): Date | null {
|
|
|
344
344
|
return null;
|
|
345
345
|
}
|
|
346
346
|
|
|
347
|
+
/** Rows per `ctx.commit` during a sync. */
|
|
348
|
+
const COMMIT_ROWS = 500;
|
|
349
|
+
|
|
347
350
|
export default class PostgresConnector extends ConnectorRuntime {
|
|
348
351
|
readonly definition: RuntimeConnectorDefinition = {
|
|
349
352
|
key: 'postgres',
|
|
@@ -466,21 +469,26 @@ export default class PostgresConnector extends ConnectorRuntime {
|
|
|
466
469
|
// to feedKey only for direct/programmatic sync calls (no feedId).
|
|
467
470
|
const originPrefix = ctx.feedId != null ? String(ctx.feedId) : ctx.feedKey;
|
|
468
471
|
|
|
469
|
-
// Map rows → events
|
|
470
|
-
|
|
471
|
-
|
|
472
|
-
for (
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
472
|
+
// Map rows → events and commit them in slices, each with the keyset
|
|
473
|
+
// checkpoint of its last row: that checkpoint resumes exactly after it,
|
|
474
|
+
// so a run cut short keeps every slice it committed.
|
|
475
|
+
for (let start = 0; start < rows.length; start += COMMIT_ROWS) {
|
|
476
|
+
const slice = rows.slice(start, start + COMMIT_ROWS);
|
|
477
|
+
const last = slice[slice.length - 1];
|
|
478
|
+
const sliceCheckpoint: PgCheckpoint = {
|
|
479
|
+
last_cursor: toCheckpointValue(last[cursorCol]),
|
|
480
|
+
last_pk: toCheckpointValue(last[pkCol]),
|
|
477
481
|
};
|
|
482
|
+
await ctx.commit(
|
|
483
|
+
slice.map((row) => this.rowToEvent(originPrefix, row, config, cursorCol, pkCol)),
|
|
484
|
+
sliceCheckpoint as unknown as Record<string, unknown>
|
|
485
|
+
);
|
|
478
486
|
}
|
|
479
487
|
|
|
480
488
|
return {
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
metadata: { items_found:
|
|
489
|
+
// A full page means rows remain past the cursor: run again now.
|
|
490
|
+
status: rows.length >= limit ? 'more' : 'complete',
|
|
491
|
+
metadata: { items_found: rows.length },
|
|
484
492
|
};
|
|
485
493
|
} finally {
|
|
486
494
|
await sql.end({ timeout: 5 });
|
|
@@ -265,10 +265,12 @@ export default class ProductHuntConnector extends ConnectorRuntime {
|
|
|
265
265
|
|
|
266
266
|
let result: ProductHuntPostsResponse;
|
|
267
267
|
try {
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
268
|
+
// A GraphQL query sent as POST: safe to repeat on a 5xx.
|
|
269
|
+
result = await http.post<ProductHuntPostsResponse>(
|
|
270
|
+
this.API_URL,
|
|
271
|
+
{ query: LIST_POSTS_QUERY, variables },
|
|
272
|
+
{ idempotent: true }
|
|
273
|
+
);
|
|
272
274
|
} catch (error) {
|
|
273
275
|
if (error instanceof HttpStatusError && error.status === 401) {
|
|
274
276
|
// PH v2 API requires auth. Fail the run instead of returning an
|
|
@@ -352,9 +354,9 @@ export default class ProductHuntConnector extends ConnectorRuntime {
|
|
|
352
354
|
last_sync_at: new Date().toISOString(),
|
|
353
355
|
};
|
|
354
356
|
|
|
357
|
+
await ctx.commit(events, checkpoint as Record<string, unknown>);
|
|
355
358
|
return {
|
|
356
|
-
|
|
357
|
-
checkpoint: checkpoint as Record<string, unknown>,
|
|
359
|
+
status: 'complete',
|
|
358
360
|
metadata: {
|
|
359
361
|
items_found: events.length,
|
|
360
362
|
},
|
|
@@ -362,9 +362,9 @@ export default class RedditConnector extends ConnectorRuntime {
|
|
|
362
362
|
pagination_token: after ?? undefined,
|
|
363
363
|
};
|
|
364
364
|
|
|
365
|
+
await ctx.commit(events, checkpoint as Record<string, unknown>);
|
|
365
366
|
return {
|
|
366
|
-
|
|
367
|
-
checkpoint: checkpoint as Record<string, unknown>,
|
|
367
|
+
status: 'complete',
|
|
368
368
|
metadata: {
|
|
369
369
|
items_found: events.length,
|
|
370
370
|
},
|