@lobu/runtime-device 20.2.0 → 21.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/component.json +1 -1
- package/dist/dependencies.bun.lock +44 -38
- package/npm-shrinkwrap.json +277 -231
- package/package.json +1 -1
- package/vendor/cli/dist/commands/_lib/apply/apply-cmd.d.ts +7 -2
- package/vendor/cli/dist/commands/_lib/apply/apply-cmd.js +77 -79
- package/vendor/cli/dist/commands/_lib/apply/client.d.ts +41 -29
- package/vendor/cli/dist/commands/_lib/apply/client.js +33 -47
- package/vendor/cli/dist/commands/_lib/apply/deployment.d.ts +1 -1
- package/vendor/cli/dist/commands/_lib/apply/deployment.js +2 -3
- package/vendor/cli/dist/commands/_lib/apply/desired-state.d.ts +47 -13
- package/vendor/cli/dist/commands/_lib/apply/desired-state.js +106 -7
- package/vendor/cli/dist/commands/_lib/apply/diff.d.ts +14 -7
- package/vendor/cli/dist/commands/_lib/apply/diff.js +42 -29
- package/vendor/cli/dist/commands/_lib/apply/map-config.js +2 -3
- package/vendor/cli/dist/commands/_lib/apply/render.js +2 -0
- package/vendor/cli/dist/commands/_lib/apply/rollback-cmd.js +3 -3
- package/vendor/cli/dist/commands/_lib/connector-loader.d.ts +5 -1
- package/vendor/cli/dist/commands/_lib/connector-loader.js +1 -1
- package/vendor/cli/dist/commands/_lib/ensure-deps-installed.d.ts +15 -26
- package/vendor/cli/dist/commands/_lib/ensure-deps-installed.js +227 -136
- package/vendor/cli/dist/commands/_lib/view-bundler.d.ts +33 -0
- package/vendor/cli/dist/commands/_lib/view-bundler.js +211 -0
- package/vendor/cli/dist/commands/_lib/view-reapply.d.ts +60 -0
- package/vendor/cli/dist/commands/_lib/view-reapply.js +182 -0
- package/vendor/cli/dist/commands/dev.d.ts +14 -5
- package/vendor/cli/dist/commands/dev.js +56 -8
- package/vendor/cli/dist/commands/init.d.ts +1 -1
- package/vendor/cli/dist/commands/init.js +4 -4
- package/vendor/cli/dist/commands/memory/_lib/schema.d.ts +0 -1
- package/vendor/cli/dist/commands/memory/_lib/schema.js +0 -9
- package/vendor/cli/dist/commands/memory/_lib/seed-cmd.js +0 -2
- package/vendor/cli/dist/config/define.d.ts +25 -8
- package/vendor/cli/dist/config/define.js +24 -0
- package/vendor/cli/dist/connectors/README.md +15 -16
- package/vendor/cli/dist/connectors/github.ts +274 -193
- package/vendor/cli/dist/connectors/google_calendar.ts +54 -160
- package/vendor/cli/dist/connectors/google_drive.ts +106 -131
- package/vendor/cli/dist/connectors/google_gmail.ts +53 -30
- package/vendor/cli/dist/connectors/hackernews.ts +2 -5
- package/vendor/cli/dist/connectors/jira.ts +71 -27
- package/vendor/cli/dist/connectors/linear.ts +55 -21
- package/vendor/cli/dist/connectors/microsoft_outlook.ts +151 -110
- package/vendor/cli/dist/connectors/postgres.ts +19 -11
- package/vendor/cli/dist/connectors/producthunt.ts +11 -7
- package/vendor/cli/dist/connectors/reddit.ts +36 -6
- package/vendor/cli/dist/connectors/rss.ts +2 -2
- package/vendor/cli/dist/connectors/whatsapp_web.ts +5 -4
- package/vendor/cli/dist/connectors/x.ts +32 -17
- package/vendor/cli/dist/connectors/youtube.ts +18 -5
- package/vendor/cli/dist/internal/connector-compiler.d.ts +1 -1
- package/vendor/cli/dist/internal/connector-compiler.js +216 -17
- package/vendor/cli/dist/providers.json +0 -1
- package/vendor/cli/dist/runtime-components.json +4 -4
- package/vendor/cli/package.json +2 -1
- package/vendor/connector-sdk/dist/connector-runtime.d.ts +4 -6
- package/vendor/connector-sdk/dist/connector-runtime.js +3 -5
- package/vendor/connector-sdk/dist/connector-types.d.ts +28 -15
- package/vendor/connector-sdk/dist/define-connector.d.ts +4 -1
- package/vendor/connector-sdk/dist/define-connector.js +4 -1
- package/vendor/connector-sdk/dist/http-client.d.ts +18 -7
- package/vendor/connector-sdk/dist/http-client.js +15 -13
- package/vendor/connector-sdk/dist/index.d.ts +1 -1
- package/vendor/connector-sdk/dist/retry.d.ts +15 -3
- package/vendor/connector-sdk/dist/retry.js +40 -64
- package/vendor/connector-sdk/package.json +1 -1
- package/vendor/connector-worker/dist/agent-turn/guest-entry.js +122 -10
- package/vendor/connector-worker/dist/agent-turn/guest.bundle.js +78 -10
- package/vendor/connector-worker/dist/agent-turn/types.d.ts +34 -4
- package/vendor/connector-worker/dist/compile/index.d.ts +21 -2
- package/vendor/connector-worker/dist/compile/index.js +41 -8
- package/vendor/connector-worker/dist/compile/source-capture.d.ts +10 -0
- package/vendor/connector-worker/dist/compile/source-capture.js +108 -0
- package/vendor/connector-worker/dist/daemon/agent-turn.js +97 -10
- package/vendor/connector-worker/dist/daemon/automation.d.ts +1 -1
- package/vendor/connector-worker/dist/daemon/client.d.ts +4 -1
- package/vendor/connector-worker/dist/daemon/client.js +3 -0
- package/vendor/connector-worker/dist/daemon/executor.d.ts +0 -1
- package/vendor/connector-worker/dist/daemon/executor.js +46 -51
- package/vendor/connector-worker/dist/daemon/start.js +6 -2
- package/vendor/connector-worker/dist/executor/interface.d.ts +19 -9
- package/vendor/connector-worker/dist/executor/isolate.js +85 -28
- package/vendor/connector-worker/dist/self-check/index.js +15 -13
- package/vendor/connector-worker/package.json +1 -1
- package/vendor/core/dist/agent-policy.js +3 -0
- package/vendor/core/dist/connector-query-errors.d.ts +3 -3
- package/vendor/core/dist/connector-query-errors.js +8 -3
- package/vendor/core/dist/contracts/tools/manage-auth-profiles.d.ts +14 -0
- package/vendor/core/dist/contracts/tools/manage-auth-profiles.js +12 -1
- package/vendor/core/dist/contracts/tools/manage-automations.d.ts +0 -10
- package/vendor/core/dist/contracts/tools/manage-automations.js +0 -33
- package/vendor/core/dist/contracts/tools/manage-classifiers.d.ts +4 -34
- package/vendor/core/dist/contracts/tools/manage-classifiers.js +13 -44
- package/vendor/core/dist/contracts/tools/manage-connections.d.ts +24 -0
- package/vendor/core/dist/contracts/tools/manage-connections.js +13 -0
- package/vendor/core/dist/contracts/tools/manage-entity-schema.d.ts +15 -25
- package/vendor/core/dist/contracts/tools/manage-entity-schema.js +3 -5
- package/vendor/core/dist/contracts/tools/manage-entity.d.ts +0 -9
- package/vendor/core/dist/contracts/tools/manage-entity.js +0 -6
- package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +11 -0
- package/vendor/core/dist/contracts/tools/manage-feeds.js +12 -1
- package/vendor/core/dist/contracts/tools/manage-operations.d.ts +47 -6
- package/vendor/core/dist/contracts/tools/manage-operations.js +36 -1
- package/vendor/core/dist/contracts/tools/manage-views.d.ts +293 -0
- package/vendor/core/dist/contracts/tools/manage-views.js +203 -0
- package/vendor/core/dist/contracts/tools/source-files-schema.d.ts +7 -0
- package/vendor/core/dist/contracts/tools/source-files-schema.js +19 -0
- package/vendor/core/dist/contracts/tools/source-files.d.ts +14 -0
- package/vendor/core/dist/contracts/tools/source-files.js +55 -0
- package/vendor/core/dist/contracts/tools/view-attach.d.ts +20 -0
- package/vendor/core/dist/contracts/tools/view-attach.js +45 -0
- package/vendor/core/dist/contracts/tools/view-content-hash.d.ts +37 -0
- package/vendor/core/dist/contracts/tools/view-content-hash.js +58 -0
- package/vendor/core/dist/contracts/tools/view-path.d.ts +18 -0
- package/vendor/core/dist/contracts/tools/view-path.js +24 -0
- package/vendor/core/dist/contracts/worker/protocol.d.ts +90 -8
- package/vendor/core/dist/contracts/worker/protocol.js +70 -10
- package/vendor/core/dist/index.d.ts +1 -1
- package/vendor/core/dist/provider-config-types.d.ts +0 -9
- package/vendor/core/dist/reserved.d.ts +3 -3
- package/vendor/core/dist/reserved.js +9 -1
- package/vendor/core/dist/sdk-compat.d.ts +1 -1
- package/vendor/core/dist/utils/secret-redaction.js +2 -0
- package/vendor/core/package.json +30 -5
- package/vendor/core/dist/contracts/tools/manage-view-templates.d.ts +0 -165
- package/vendor/core/dist/contracts/tools/manage-view-templates.js +0 -130
|
@@ -100,6 +100,8 @@ interface GmailCheckpoint {
|
|
|
100
100
|
/** Serialized search + person filter the two fields below were produced under. */
|
|
101
101
|
scope?: string;
|
|
102
102
|
last_sync_at?: string;
|
|
103
|
+
/** Undecodable threads, keyed by thread ID; retained until a later successful read. */
|
|
104
|
+
skipped_threads?: Record<string, { message_id: string; error: string }>;
|
|
103
105
|
/** Set while a window is only part-walked: the run hit `max_results`, or a page came back empty with a cursor. */
|
|
104
106
|
pending?: {
|
|
105
107
|
query: string;
|
|
@@ -108,6 +110,12 @@ interface GmailCheckpoint {
|
|
|
108
110
|
};
|
|
109
111
|
}
|
|
110
112
|
|
|
113
|
+
class GmailBodyDecodeError extends Error {
|
|
114
|
+
constructor(readonly messageId: string, encoding: string, cause: unknown) {
|
|
115
|
+
super(`Gmail body decode (${encoding}): ${cause instanceof Error ? cause.message : String(cause)}`, { cause });
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
|
|
111
119
|
interface GmailConfig {
|
|
112
120
|
/**
|
|
113
121
|
* Gmail search scope shared by sync and direct source reads. Takes precedence
|
|
@@ -449,7 +457,7 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
449
457
|
// sync
|
|
450
458
|
// -------------------------------------------------------------------------
|
|
451
459
|
|
|
452
|
-
private async syncFeed(ctx: SyncContext<GmailCheckpoint, GmailConfig>): Promise<SyncResult
|
|
460
|
+
private async syncFeed(ctx: SyncContext<GmailCheckpoint, GmailConfig>): Promise<SyncResult> {
|
|
453
461
|
const syncStartedAt = new Date();
|
|
454
462
|
const token = ctx.credentials?.accessToken;
|
|
455
463
|
if (!token) {
|
|
@@ -485,7 +493,23 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
485
493
|
// first page. `origin_id` makes that replay a supersede, not a duplicate.
|
|
486
494
|
let staleCursorRecoverable = pageToken !== undefined;
|
|
487
495
|
const http = this.createClient(token);
|
|
488
|
-
const
|
|
496
|
+
const skippedThreads = { ...checkpoint.skipped_threads };
|
|
497
|
+
let itemsFound = 0;
|
|
498
|
+
let itemsSkipped = 0;
|
|
499
|
+
// A part-walked window keeps the previous `last_sync_at`: it may not advance
|
|
500
|
+
// until the whole window has been walked, or the unread tail is skipped.
|
|
501
|
+
const checkpointAt = (cursor: string | undefined): GmailCheckpoint => {
|
|
502
|
+
const next: GmailCheckpoint = cursor
|
|
503
|
+
? {
|
|
504
|
+
schema_version: 3,
|
|
505
|
+
scope: scopeKey,
|
|
506
|
+
...(resumable && checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
|
|
507
|
+
pending: { query, started_at: windowStart, page_token: cursor },
|
|
508
|
+
}
|
|
509
|
+
: { schema_version: 3, scope: scopeKey, last_sync_at: windowStart };
|
|
510
|
+
if (Object.keys(skippedThreads).length) next.skipped_threads = { ...skippedThreads };
|
|
511
|
+
return next;
|
|
512
|
+
};
|
|
489
513
|
// Bounds the threads FETCHED per run (each costs at least one API call),
|
|
490
514
|
// independent of how many survive the person filter — a narrow feed must not
|
|
491
515
|
// scan the whole window just because most threads are rejected. It also sizes
|
|
@@ -513,6 +537,7 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
513
537
|
staleCursorRecoverable = false;
|
|
514
538
|
const listData = (await listResponse.json()) as GmailThreadListResponse;
|
|
515
539
|
const threads = listData.threads ?? [];
|
|
540
|
+
const events: EventEnvelope[] = [];
|
|
516
541
|
for (const threadStub of threads) {
|
|
517
542
|
inspected++;
|
|
518
543
|
try {
|
|
@@ -562,6 +587,13 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
562
587
|
...(attribution.fromName ? { from_name: attribution.fromName } : {}),
|
|
563
588
|
},
|
|
564
589
|
});
|
|
590
|
+
delete skippedThreads[threadStub.id];
|
|
591
|
+
} catch (error) {
|
|
592
|
+
// Only decoding failures are permanent for these bytes. Fetch and
|
|
593
|
+
// conversion failures must still retry the window without advancing.
|
|
594
|
+
if (!(error instanceof GmailBodyDecodeError)) throw error;
|
|
595
|
+
skippedThreads[threadStub.id] = { message_id: error.messageId, error: error.message };
|
|
596
|
+
itemsSkipped++;
|
|
565
597
|
} finally {
|
|
566
598
|
await sleep(this.RATE_LIMIT_MS);
|
|
567
599
|
}
|
|
@@ -570,31 +602,18 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
570
602
|
throw new Error('Gmail returned a repeated page token');
|
|
571
603
|
}
|
|
572
604
|
pageToken = listData.nextPageToken;
|
|
573
|
-
//
|
|
574
|
-
//
|
|
575
|
-
|
|
576
|
-
|
|
605
|
+
// The page commits with the cursor of the next one, so a run cut short
|
|
606
|
+
// resumes where it stopped; the last page commits the finished window.
|
|
607
|
+
await ctx.commit(events, checkpointAt(pageToken));
|
|
608
|
+
itemsFound += events.length;
|
|
609
|
+
const metadata = { items_found: itemsFound, items_skipped: itemsSkipped };
|
|
610
|
+
if (!pageToken) return { status: 'complete', metadata };
|
|
611
|
+
// An empty page with a cursor is still incomplete. Its cursor is saved
|
|
612
|
+
// and the next run continues immediately rather than reporting the
|
|
613
|
+
// fixed window as caught up.
|
|
614
|
+
if (threads.length === 0) return { status: 'more', metadata };
|
|
615
|
+
if (inspected >= maxResults) return { status: 'more', metadata };
|
|
577
616
|
}
|
|
578
|
-
|
|
579
|
-
events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
|
|
580
|
-
// A part-walked window keeps the previous `last_sync_at`: it may not advance
|
|
581
|
-
// until the whole window has been walked, or the unread tail is skipped.
|
|
582
|
-
const newCheckpoint: GmailCheckpoint = pageToken
|
|
583
|
-
? {
|
|
584
|
-
schema_version: 3,
|
|
585
|
-
scope: scopeKey,
|
|
586
|
-
...(resumable && checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
|
|
587
|
-
pending: { query, started_at: windowStart, page_token: pageToken },
|
|
588
|
-
}
|
|
589
|
-
: { schema_version: 3, scope: scopeKey, last_sync_at: windowStart };
|
|
590
|
-
|
|
591
|
-
return {
|
|
592
|
-
events,
|
|
593
|
-
checkpoint: newCheckpoint,
|
|
594
|
-
metadata: {
|
|
595
|
-
items_found: events.length,
|
|
596
|
-
},
|
|
597
|
-
};
|
|
598
617
|
}
|
|
599
618
|
|
|
600
619
|
// -------------------------------------------------------------------------
|
|
@@ -1356,7 +1375,7 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
1356
1375
|
if (mimeType !== 'text/plain' && mimeType !== 'text/html') return '';
|
|
1357
1376
|
let text = '';
|
|
1358
1377
|
if (payload.body?.data) {
|
|
1359
|
-
text = this.base64UrlDecode(payload.body.data, payload);
|
|
1378
|
+
text = this.base64UrlDecode(payload.body.data, payload, messageId);
|
|
1360
1379
|
} else if (payload.body?.attachmentId) {
|
|
1361
1380
|
// Gmail may externalize the body itself, even under format=full. A
|
|
1362
1381
|
// failed fetch must not silently replace that body with its alternative.
|
|
@@ -1364,7 +1383,7 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
1364
1383
|
if (!response.ok) throw new Error(`Gmail message body error (${response.status}): ${await response.text()}`);
|
|
1365
1384
|
const body = (await response.json()) as { data?: string };
|
|
1366
1385
|
if (typeof body.data !== 'string') throw new Error('Gmail returned a message body without data');
|
|
1367
|
-
text = this.base64UrlDecode(body.data, payload);
|
|
1386
|
+
text = this.base64UrlDecode(body.data, payload, messageId);
|
|
1368
1387
|
}
|
|
1369
1388
|
if (!text && payload.body?.size) throw new Error('Gmail returned a nonempty message body without data');
|
|
1370
1389
|
// Convert each HTML leaf after charset decoding, preserving plain parts,
|
|
@@ -1374,14 +1393,18 @@ export default class GmailConnector extends ConnectorRuntime<GmailCheckpoint, Gm
|
|
|
1374
1393
|
return text.trim() ? text : '';
|
|
1375
1394
|
}
|
|
1376
1395
|
|
|
1377
|
-
private base64UrlDecode(data: string, payload: GmailMessagePayload): string {
|
|
1396
|
+
private base64UrlDecode(data: string, payload: GmailMessagePayload, messageId: string): string {
|
|
1378
1397
|
const padded = data.replace(/-/g, '+').replace(/_/g, '/');
|
|
1379
1398
|
const contentType = payload.headers?.find((header) => header.name.toLowerCase() === 'content-type')?.value;
|
|
1380
1399
|
const charset = contentType?.match(/;\s*charset\s*=\s*(?:"([^"]*)"|'([^']*)'|([^;\s]+))/i);
|
|
1381
1400
|
const encoding = (charset?.[1] ?? charset?.[2] ?? charset?.[3] ?? 'utf-8').trim();
|
|
1382
1401
|
// MIME bodies contain bytes in the part's declared charset. Invalid bytes
|
|
1383
1402
|
// or unsupported labels must fail the read instead of persisting corruption.
|
|
1384
|
-
|
|
1403
|
+
try {
|
|
1404
|
+
return new TextDecoder(encoding, { fatal: true }).decode(Buffer.from(padded, 'base64'));
|
|
1405
|
+
} catch (cause) {
|
|
1406
|
+
throw new GmailBodyDecodeError(messageId, encoding, cause);
|
|
1407
|
+
}
|
|
1385
1408
|
}
|
|
1386
1409
|
|
|
1387
1410
|
private base64UrlEncode(str: string): string {
|
|
@@ -376,11 +376,8 @@ export default class HackerNewsConnector extends ConnectorRuntime {
|
|
|
376
376
|
await this.enrichStoriesWithExternalContent(events, deadline);
|
|
377
377
|
}
|
|
378
378
|
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
checkpoint: { last_sync_at: new Date().toISOString() },
|
|
382
|
-
metadata: { items_found: events.length },
|
|
383
|
-
};
|
|
379
|
+
await ctx.commit(events, { last_sync_at: new Date().toISOString() });
|
|
380
|
+
return { status: 'complete', metadata: { items_found: events.length } };
|
|
384
381
|
}
|
|
385
382
|
|
|
386
383
|
// -------------------------------------------------------------------------
|
|
@@ -12,7 +12,7 @@ import {
|
|
|
12
12
|
type EventEnvelope,
|
|
13
13
|
type FeedReadContext,
|
|
14
14
|
type FeedReadResult,
|
|
15
|
-
|
|
15
|
+
HttpStatusError,
|
|
16
16
|
requireBearerClient,
|
|
17
17
|
type RuntimeConnectorDefinition,
|
|
18
18
|
type SyncContext,
|
|
@@ -64,7 +64,15 @@ interface JiraConfig {
|
|
|
64
64
|
}
|
|
65
65
|
|
|
66
66
|
interface JiraCheckpoint {
|
|
67
|
+
/** When the last complete traversal began; the next one fetches issues updated since. */
|
|
67
68
|
last_sync_at?: string;
|
|
69
|
+
/**
|
|
70
|
+
* A traversal that stopped at the per-run page cap: the fixed `updated`
|
|
71
|
+
* window it walks and the `nextPageToken` of the first page not yet stored,
|
|
72
|
+
* so the next run continues instead of re-reading the newest issues and
|
|
73
|
+
* never reaching the rest.
|
|
74
|
+
*/
|
|
75
|
+
pending?: { window: { start: string; end: string }; page_token: string };
|
|
68
76
|
}
|
|
69
77
|
|
|
70
78
|
interface JiraUser {
|
|
@@ -519,52 +527,88 @@ export default class JiraConnector extends ConnectorRuntime<JiraCheckpoint, Jira
|
|
|
519
527
|
// Sync into local memory.
|
|
520
528
|
// -------------------------------------------------------------------------
|
|
521
529
|
|
|
522
|
-
private async syncFeed(ctx: SyncContext<JiraCheckpoint, JiraConfig>): Promise<SyncResult
|
|
530
|
+
private async syncFeed(ctx: SyncContext<JiraCheckpoint, JiraConfig>): Promise<SyncResult> {
|
|
523
531
|
const base = await this.restBase(ctx.config, ctx.sessionState, ctx.credentials);
|
|
524
532
|
const http = this.client(ctx.credentials);
|
|
533
|
+
const checkpoint = ctx.checkpoint ?? {};
|
|
525
534
|
const lookbackDays = ctx.config.lookback_days ?? 365;
|
|
535
|
+
// Fixed per traversal: issues updated since the last complete traversal
|
|
536
|
+
// began, up to when this one began. An issue edited while the walk runs
|
|
537
|
+
// falls in the next window, and `origin_id` makes a re-report a supersede.
|
|
538
|
+
const window = checkpoint.pending?.window ?? {
|
|
539
|
+
start: checkpoint.last_sync_at ?? new Date(0).toISOString(),
|
|
540
|
+
end: new Date().toISOString(),
|
|
541
|
+
};
|
|
526
542
|
const jql = buildJiraJql({
|
|
527
543
|
baseQuery: asString(ctx.config.query) ?? asString(ctx.config.jql) ?? '',
|
|
528
544
|
defaultWhenEmpty: `updated >= -${lookbackDays}d`,
|
|
545
|
+
window,
|
|
529
546
|
});
|
|
530
547
|
|
|
531
|
-
const events: EventEnvelope[] = [];
|
|
532
|
-
|
|
533
548
|
// `/rest/api/3/search` was removed by Atlassian (CHANGE-2046); the
|
|
534
549
|
// replacement `/search/jql` paginates with an opaque nextPageToken and
|
|
535
550
|
// returns no total — iterate until the token is absent.
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
539
|
-
|
|
540
|
-
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
551
|
+
let pageToken = checkpoint.pending?.page_token;
|
|
552
|
+
// A stored token may be retired between runs. The first request of the run
|
|
553
|
+
// — the only one using a token from a previous run — may restart the same
|
|
554
|
+
// window from its first page instead of wedging the feed.
|
|
555
|
+
let staleCursorRecoverable = pageToken !== undefined;
|
|
556
|
+
let found = 0;
|
|
557
|
+
const seenTokens = new Set<string>();
|
|
558
|
+
for (let page = 0; page < this.MAX_PAGES; page++) {
|
|
559
|
+
const params = new URLSearchParams({
|
|
560
|
+
jql,
|
|
561
|
+
maxResults: String(this.PAGE_SIZE),
|
|
562
|
+
fields: ISSUE_FIELDS,
|
|
563
|
+
});
|
|
564
|
+
if (pageToken) {
|
|
565
|
+
seenTokens.add(pageToken);
|
|
566
|
+
params.set('nextPageToken', pageToken);
|
|
567
|
+
}
|
|
568
|
+
let data: JiraSearchResponse;
|
|
569
|
+
try {
|
|
570
|
+
data = await http.json<JiraSearchResponse>(
|
|
545
571
|
`${base}/search/jql?${params.toString()}`,
|
|
546
572
|
{ method: 'GET', headers: { Accept: 'application/json' } },
|
|
547
573
|
);
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
574
|
+
} catch (error) {
|
|
575
|
+
if (staleCursorRecoverable && error instanceof HttpStatusError && error.status === 400) {
|
|
576
|
+
staleCursorRecoverable = false;
|
|
577
|
+
pageToken = undefined;
|
|
578
|
+
seenTokens.clear();
|
|
579
|
+
page--;
|
|
580
|
+
continue;
|
|
581
|
+
}
|
|
582
|
+
throw error;
|
|
583
|
+
}
|
|
584
|
+
staleCursorRecoverable = false;
|
|
552
585
|
|
|
553
|
-
|
|
586
|
+
const issues = data.issues ?? [];
|
|
587
|
+
const events: EventEnvelope[] = [];
|
|
554
588
|
for (const issue of issues) {
|
|
555
589
|
const event = this.issueEvent(issue, ctx.config);
|
|
556
590
|
if (event) events.push(event);
|
|
557
591
|
}
|
|
558
|
-
|
|
559
|
-
// returned — guards against a degenerate self-referential cursor.
|
|
560
|
-
if (issues.length === 0) break;
|
|
561
|
-
}
|
|
592
|
+
found += events.length;
|
|
562
593
|
|
|
563
|
-
|
|
564
|
-
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
594
|
+
if (!data.nextPageToken) {
|
|
595
|
+
await ctx.commit(events, { last_sync_at: window.end });
|
|
596
|
+
return { status: 'complete', metadata: { items_found: found } };
|
|
597
|
+
}
|
|
598
|
+
if (seenTokens.has(data.nextPageToken)) {
|
|
599
|
+
throw new Error('Jira returned a repeated page token.');
|
|
600
|
+
}
|
|
601
|
+
pageToken = data.nextPageToken;
|
|
602
|
+
await ctx.commit(events, {
|
|
603
|
+
...(checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
|
|
604
|
+
pending: { window, page_token: pageToken },
|
|
605
|
+
});
|
|
606
|
+
if (issues.length === 0) {
|
|
607
|
+
return { status: 'more', metadata: { items_found: found } };
|
|
608
|
+
}
|
|
609
|
+
}
|
|
610
|
+
// The page cap stopped the walk; its committed token resumes it next run.
|
|
611
|
+
return { status: 'more', metadata: { items_found: found } };
|
|
568
612
|
}
|
|
569
613
|
|
|
570
614
|
// -------------------------------------------------------------------------
|
|
@@ -40,7 +40,15 @@ interface LinearConfig {
|
|
|
40
40
|
}
|
|
41
41
|
|
|
42
42
|
interface LinearCheckpoint {
|
|
43
|
+
/** When the last complete traversal began; the next one fetches issues updated since. */
|
|
43
44
|
last_sync_at?: string;
|
|
45
|
+
/**
|
|
46
|
+
* A traversal that stopped at the per-run page cap. `since`/`started_at`
|
|
47
|
+
* are the fixed bounds it walks and `cursor` the `endCursor` of the last page
|
|
48
|
+
* stored, so the next run continues past it instead of re-reading the newest
|
|
49
|
+
* issues and never reaching the rest.
|
|
50
|
+
*/
|
|
51
|
+
pending?: { since?: string; started_at: string; cursor: string };
|
|
44
52
|
}
|
|
45
53
|
|
|
46
54
|
interface LinearUser {
|
|
@@ -297,7 +305,7 @@ export default class LinearConnector extends ConnectorRuntime<LinearCheckpoint,
|
|
|
297
305
|
pageInfo?: { hasNextPage?: boolean; endCursor?: string | null };
|
|
298
306
|
nodes?: LinearIssueNode[];
|
|
299
307
|
};
|
|
300
|
-
} = await this.graphql(ctx.credentials, gql);
|
|
308
|
+
} = await this.graphql(ctx.credentials, gql, undefined, { idempotent: true });
|
|
301
309
|
|
|
302
310
|
const rows = (response.issues?.nodes ?? [])
|
|
303
311
|
.map((node) => this.issueRow(node))
|
|
@@ -323,16 +331,24 @@ export default class LinearConnector extends ConnectorRuntime<LinearCheckpoint,
|
|
|
323
331
|
// Sync into local memory.
|
|
324
332
|
// -------------------------------------------------------------------------
|
|
325
333
|
|
|
326
|
-
private async syncFeed(ctx: SyncContext<LinearCheckpoint, LinearConfig>): Promise<SyncResult
|
|
327
|
-
const
|
|
328
|
-
|
|
329
|
-
|
|
334
|
+
private async syncFeed(ctx: SyncContext<LinearCheckpoint, LinearConfig>): Promise<SyncResult> {
|
|
335
|
+
const checkpoint = ctx.checkpoint ?? {};
|
|
336
|
+
// Fixed per traversal: issues updated after the last complete traversal
|
|
337
|
+
// began (none on the first), walked until the listing ends. An issue edited
|
|
338
|
+
// while the walk runs is re-reported by the next traversal, and `origin_id`
|
|
339
|
+
// makes that a supersede, not a duplicate.
|
|
340
|
+
const since = checkpoint.pending ? checkpoint.pending.since : checkpoint.last_sync_at;
|
|
341
|
+
const startedAt = checkpoint.pending?.started_at ?? new Date().toISOString();
|
|
342
|
+
let cursor: string | null = checkpoint.pending?.cursor ?? null;
|
|
330
343
|
|
|
331
|
-
const filter =
|
|
332
|
-
|
|
333
|
-
:
|
|
344
|
+
const filter = buildLinearIssueFilter({
|
|
345
|
+
teamKey: asString(ctx.config.team_key),
|
|
346
|
+
updatedAfterIso: since,
|
|
347
|
+
updatedBeforeIso: startedAt,
|
|
348
|
+
});
|
|
334
349
|
|
|
335
|
-
|
|
350
|
+
let found = 0;
|
|
351
|
+
for (let pages = 0; pages < this.MAX_PAGES; pages++) {
|
|
336
352
|
const after: string = cursor ? `, after: ${JSON.stringify(cursor)}` : '';
|
|
337
353
|
const query: string = `
|
|
338
354
|
query {
|
|
@@ -348,25 +364,36 @@ export default class LinearConnector extends ConnectorRuntime<LinearCheckpoint,
|
|
|
348
364
|
pageInfo?: { hasNextPage?: boolean; endCursor?: string | null };
|
|
349
365
|
nodes?: LinearIssueNode[];
|
|
350
366
|
};
|
|
351
|
-
}>(ctx.credentials, query);
|
|
367
|
+
}>(ctx.credentials, query, undefined, { idempotent: true });
|
|
352
368
|
|
|
353
|
-
const nodes = response.issues?.nodes
|
|
369
|
+
const nodes = response.issues?.nodes;
|
|
370
|
+
const pageInfo = response.issues?.pageInfo;
|
|
371
|
+
if (
|
|
372
|
+
!Array.isArray(nodes) ||
|
|
373
|
+
typeof pageInfo?.hasNextPage !== 'boolean' ||
|
|
374
|
+
(pageInfo.hasNextPage && !pageInfo.endCursor)
|
|
375
|
+
) {
|
|
376
|
+
throw new Error('Linear did not return a valid page cursor/exhaustion state.');
|
|
377
|
+
}
|
|
378
|
+
const events: EventEnvelope[] = [];
|
|
354
379
|
for (const node of nodes) {
|
|
355
380
|
const event = this.issueEvent(node);
|
|
356
381
|
if (event) events.push(event);
|
|
357
382
|
}
|
|
383
|
+
found += events.length;
|
|
358
384
|
|
|
359
|
-
|
|
360
|
-
|
|
361
|
-
|
|
385
|
+
if (!pageInfo?.hasNextPage || !pageInfo.endCursor) {
|
|
386
|
+
await ctx.commit(events, { last_sync_at: startedAt });
|
|
387
|
+
return { status: 'complete', metadata: { items_found: found } };
|
|
388
|
+
}
|
|
362
389
|
cursor = pageInfo.endCursor;
|
|
390
|
+
await ctx.commit(events, {
|
|
391
|
+
...(checkpoint.last_sync_at ? { last_sync_at: checkpoint.last_sync_at } : {}),
|
|
392
|
+
pending: { ...(since ? { since } : {}), started_at: startedAt, cursor },
|
|
393
|
+
});
|
|
363
394
|
}
|
|
364
|
-
|
|
365
|
-
return {
|
|
366
|
-
events,
|
|
367
|
-
checkpoint: { last_sync_at: new Date().toISOString() },
|
|
368
|
-
metadata: { items_found: events.length },
|
|
369
|
-
};
|
|
395
|
+
// The page cap stopped the walk; its committed cursor resumes it next run.
|
|
396
|
+
return { status: 'more', metadata: { items_found: found } };
|
|
370
397
|
}
|
|
371
398
|
|
|
372
399
|
// -------------------------------------------------------------------------
|
|
@@ -463,10 +490,16 @@ export default class LinearConnector extends ConnectorRuntime<LinearCheckpoint,
|
|
|
463
490
|
// GraphQL transport
|
|
464
491
|
// -------------------------------------------------------------------------
|
|
465
492
|
|
|
493
|
+
/**
|
|
494
|
+
* `idempotent` marks a read: a GraphQL query goes out as POST, so without it
|
|
495
|
+
* the client never repeats the request on a 5xx. Leave it off for mutations —
|
|
496
|
+
* a 5xx does not say whether the webhook was already created.
|
|
497
|
+
*/
|
|
466
498
|
private async graphql<T>(
|
|
467
499
|
credentials: SyncCredentials | null,
|
|
468
500
|
query: string,
|
|
469
|
-
variables?: Record<string, unknown
|
|
501
|
+
variables?: Record<string, unknown>,
|
|
502
|
+
options: { idempotent?: boolean } = {}
|
|
470
503
|
): Promise<T> {
|
|
471
504
|
const http = requireBearerClient(credentials, {
|
|
472
505
|
errorPrefix: 'Linear API',
|
|
@@ -478,6 +511,7 @@ export default class LinearConnector extends ConnectorRuntime<LinearCheckpoint,
|
|
|
478
511
|
method: 'POST',
|
|
479
512
|
headers: { 'Content-Type': 'application/json' },
|
|
480
513
|
body: JSON.stringify({ query, variables: variables ?? {} }),
|
|
514
|
+
idempotent: options.idempotent,
|
|
481
515
|
}
|
|
482
516
|
);
|
|
483
517
|
|