@lobu/runtime-device 20.2.0 → 21.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. package/dist/component.json +1 -1
  2. package/dist/dependencies.bun.lock +42 -36
  3. package/npm-shrinkwrap.json +270 -224
  4. package/package.json +1 -1
  5. package/vendor/cli/dist/commands/_lib/apply/apply-cmd.d.ts +7 -2
  6. package/vendor/cli/dist/commands/_lib/apply/apply-cmd.js +77 -75
  7. package/vendor/cli/dist/commands/_lib/apply/client.d.ts +40 -25
  8. package/vendor/cli/dist/commands/_lib/apply/client.js +31 -39
  9. package/vendor/cli/dist/commands/_lib/apply/deployment.d.ts +1 -1
  10. package/vendor/cli/dist/commands/_lib/apply/deployment.js +2 -2
  11. package/vendor/cli/dist/commands/_lib/apply/desired-state.d.ts +47 -11
  12. package/vendor/cli/dist/commands/_lib/apply/desired-state.js +106 -7
  13. package/vendor/cli/dist/commands/_lib/apply/diff.d.ts +14 -6
  14. package/vendor/cli/dist/commands/_lib/apply/diff.js +42 -23
  15. package/vendor/cli/dist/commands/_lib/apply/map-config.js +2 -3
  16. package/vendor/cli/dist/commands/_lib/apply/render.js +2 -0
  17. package/vendor/cli/dist/commands/_lib/apply/rollback-cmd.js +3 -3
  18. package/vendor/cli/dist/commands/_lib/connector-loader.d.ts +5 -1
  19. package/vendor/cli/dist/commands/_lib/connector-loader.js +1 -1
  20. package/vendor/cli/dist/commands/_lib/ensure-deps-installed.d.ts +15 -26
  21. package/vendor/cli/dist/commands/_lib/ensure-deps-installed.js +227 -136
  22. package/vendor/cli/dist/commands/_lib/view-bundler.d.ts +33 -0
  23. package/vendor/cli/dist/commands/_lib/view-bundler.js +207 -0
  24. package/vendor/cli/dist/commands/_lib/view-reapply.d.ts +60 -0
  25. package/vendor/cli/dist/commands/_lib/view-reapply.js +182 -0
  26. package/vendor/cli/dist/commands/dev.d.ts +14 -5
  27. package/vendor/cli/dist/commands/dev.js +56 -8
  28. package/vendor/cli/dist/commands/init.d.ts +1 -1
  29. package/vendor/cli/dist/commands/init.js +4 -4
  30. package/vendor/cli/dist/config/define.d.ts +25 -8
  31. package/vendor/cli/dist/config/define.js +24 -0
  32. package/vendor/cli/dist/connectors/README.md +15 -16
  33. package/vendor/cli/dist/connectors/github.ts +15 -9
  34. package/vendor/cli/dist/connectors/google_calendar.ts +21 -31
  35. package/vendor/cli/dist/connectors/google_drive.ts +106 -131
  36. package/vendor/cli/dist/connectors/google_gmail.ts +53 -30
  37. package/vendor/cli/dist/connectors/hackernews.ts +2 -5
  38. package/vendor/cli/dist/connectors/jira.ts +62 -27
  39. package/vendor/cli/dist/connectors/linear.ts +55 -21
  40. package/vendor/cli/dist/connectors/microsoft_outlook.ts +151 -110
  41. package/vendor/cli/dist/connectors/postgres.ts +19 -11
  42. package/vendor/cli/dist/connectors/producthunt.ts +8 -6
  43. package/vendor/cli/dist/connectors/reddit.ts +2 -2
  44. package/vendor/cli/dist/connectors/rss.ts +2 -2
  45. package/vendor/cli/dist/connectors/whatsapp_web.ts +5 -4
  46. package/vendor/cli/dist/connectors/x.ts +32 -17
  47. package/vendor/cli/dist/connectors/youtube.ts +18 -5
  48. package/vendor/cli/dist/internal/connector-compiler.d.ts +1 -1
  49. package/vendor/cli/dist/internal/connector-compiler.js +216 -17
  50. package/vendor/cli/dist/runtime-components.json +4 -4
  51. package/vendor/cli/package.json +2 -1
  52. package/vendor/connector-sdk/dist/connector-runtime.d.ts +4 -6
  53. package/vendor/connector-sdk/dist/connector-runtime.js +3 -5
  54. package/vendor/connector-sdk/dist/connector-types.d.ts +28 -13
  55. package/vendor/connector-sdk/dist/define-connector.d.ts +4 -1
  56. package/vendor/connector-sdk/dist/define-connector.js +4 -1
  57. package/vendor/connector-sdk/dist/http-client.d.ts +18 -7
  58. package/vendor/connector-sdk/dist/http-client.js +15 -13
  59. package/vendor/connector-sdk/dist/index.d.ts +1 -1
  60. package/vendor/connector-sdk/dist/retry.d.ts +15 -3
  61. package/vendor/connector-sdk/dist/retry.js +40 -64
  62. package/vendor/connector-sdk/package.json +1 -1
  63. package/vendor/connector-worker/dist/agent-turn/guest-entry.js +122 -10
  64. package/vendor/connector-worker/dist/agent-turn/guest.bundle.js +78 -10
  65. package/vendor/connector-worker/dist/agent-turn/types.d.ts +34 -4
  66. package/vendor/connector-worker/dist/compile/index.d.ts +21 -2
  67. package/vendor/connector-worker/dist/compile/index.js +41 -8
  68. package/vendor/connector-worker/dist/compile/source-capture.d.ts +10 -0
  69. package/vendor/connector-worker/dist/compile/source-capture.js +108 -0
  70. package/vendor/connector-worker/dist/daemon/agent-turn.js +97 -10
  71. package/vendor/connector-worker/dist/daemon/automation.d.ts +1 -1
  72. package/vendor/connector-worker/dist/daemon/executor.d.ts +0 -1
  73. package/vendor/connector-worker/dist/daemon/executor.js +42 -51
  74. package/vendor/connector-worker/dist/daemon/start.js +5 -1
  75. package/vendor/connector-worker/dist/executor/interface.d.ts +12 -8
  76. package/vendor/connector-worker/dist/executor/isolate.js +45 -28
  77. package/vendor/connector-worker/dist/self-check/index.js +15 -13
  78. package/vendor/connector-worker/package.json +1 -1
  79. package/vendor/core/dist/agent-policy.js +3 -0
  80. package/vendor/core/dist/connector-query-errors.d.ts +3 -3
  81. package/vendor/core/dist/connector-query-errors.js +8 -3
  82. package/vendor/core/dist/contracts/tools/manage-connections.d.ts +24 -0
  83. package/vendor/core/dist/contracts/tools/manage-connections.js +13 -0
  84. package/vendor/core/dist/contracts/tools/manage-entity-schema.d.ts +15 -25
  85. package/vendor/core/dist/contracts/tools/manage-entity-schema.js +3 -5
  86. package/vendor/core/dist/contracts/tools/manage-feeds.d.ts +11 -0
  87. package/vendor/core/dist/contracts/tools/manage-feeds.js +12 -1
  88. package/vendor/core/dist/contracts/tools/manage-operations.d.ts +47 -6
  89. package/vendor/core/dist/contracts/tools/manage-operations.js +36 -1
  90. package/vendor/core/dist/contracts/tools/manage-views.d.ts +293 -0
  91. package/vendor/core/dist/contracts/tools/manage-views.js +203 -0
  92. package/vendor/core/dist/contracts/tools/source-files-schema.d.ts +7 -0
  93. package/vendor/core/dist/contracts/tools/source-files-schema.js +19 -0
  94. package/vendor/core/dist/contracts/tools/source-files.d.ts +14 -0
  95. package/vendor/core/dist/contracts/tools/source-files.js +55 -0
  96. package/vendor/core/dist/contracts/tools/view-content-hash.d.ts +37 -0
  97. package/vendor/core/dist/contracts/tools/view-content-hash.js +58 -0
  98. package/vendor/core/dist/contracts/tools/view-path.d.ts +18 -0
  99. package/vendor/core/dist/contracts/tools/view-path.js +24 -0
  100. package/vendor/core/dist/contracts/worker/protocol.d.ts +73 -8
  101. package/vendor/core/dist/contracts/worker/protocol.js +58 -9
  102. package/vendor/core/dist/reserved.d.ts +3 -3
  103. package/vendor/core/dist/reserved.js +9 -1
  104. package/vendor/core/dist/utils/secret-redaction.js +2 -0
  105. package/vendor/core/package.json +25 -5
  106. package/vendor/core/dist/contracts/tools/manage-view-templates.d.ts +0 -165
  107. package/vendor/core/dist/contracts/tools/manage-view-templates.js +0 -130
@@ -181,9 +181,9 @@ export default class RSSConnector extends ConnectorRuntime {
181
181
  last_published_at: latestPublishedAt,
182
182
  };
183
183
 
184
+ await ctx.commit(events, newCheckpoint as unknown as Record<string, unknown>);
184
185
  return {
185
- events,
186
- checkpoint: newCheckpoint as unknown as Record<string, unknown>,
186
+ status: 'complete',
187
187
  metadata: {
188
188
  items_found: events.length,
189
189
  feeds_fetched: feedUrls.length - fetchErrors.length,
@@ -937,7 +937,7 @@ export default class WhatsAppWebConnector extends ConnectorRuntime<
937
937
 
938
938
  private async syncMessages(
939
939
  ctx: SyncContext<BrowserCheckpoint, WhatsAppWebConfig>
940
- ): Promise<SyncResult<BrowserCheckpoint>> {
940
+ ): Promise<SyncResult> {
941
941
  const dispatcher = requireExtensionDispatcher(ctx);
942
942
  const tabId = await readyWhatsAppTab(dispatcher);
943
943
  return await this.collectMessages(ctx, dispatcher, tabId);
@@ -954,7 +954,7 @@ export default class WhatsAppWebConnector extends ConnectorRuntime<
954
954
  ctx: SyncContext<BrowserCheckpoint, WhatsAppWebConfig>,
955
955
  dispatcher: ChromeActionDispatcher,
956
956
  tabId: number
957
- ): Promise<SyncResult<BrowserCheckpoint>> {
957
+ ): Promise<SyncResult> {
958
958
  try {
959
959
  return await this.collectMessagesInner(ctx, dispatcher, tabId);
960
960
  } catch (error) {
@@ -968,7 +968,7 @@ export default class WhatsAppWebConnector extends ConnectorRuntime<
968
968
  ctx: SyncContext<BrowserCheckpoint, WhatsAppWebConfig>,
969
969
  dispatcher: ChromeActionDispatcher,
970
970
  tabId: number
971
- ): Promise<SyncResult<BrowserCheckpoint>> {
971
+ ): Promise<SyncResult> {
972
972
  const { checkpoint, request } = buildCollectionPlan({
973
973
  checkpoint: ctx.checkpoint as Record<string, unknown> | null,
974
974
  config: ctx.config as Record<string, unknown>,
@@ -1144,7 +1144,8 @@ export default class WhatsAppWebConnector extends ConnectorRuntime<
1144
1144
  nextCheckpoint.diagnostics =
1145
1145
  Object.keys(diagnostics).length > 0 ? diagnostics : undefined;
1146
1146
 
1147
- return { events, checkpoint: nextCheckpoint };
1147
+ await ctx.commit(events, nextCheckpoint as unknown as BrowserCheckpoint);
1148
+ return { status: "complete" };
1148
1149
  }
1149
1150
 
1150
1151
  async execute(ctx: ActionContext): Promise<ActionResult> {
@@ -47,6 +47,13 @@ import {
47
47
  type SyncContext,
48
48
  type SyncResult,
49
49
  } from "@lobu/connector-sdk";
50
+
51
+ /** What one X sync gathers before it is committed. */
52
+ type XSyncPage = {
53
+ events: EventEnvelope[];
54
+ checkpoint: Record<string, unknown>;
55
+ metadata?: SyncResult["metadata"];
56
+ };
50
57
  import { X_IDENTITY, normalizeXHandle } from "./x-identity.js";
51
58
 
52
59
  /** OAuth scopes needed per feed for the API path (not used to pause browser-capable feeds). */
@@ -1061,7 +1068,7 @@ export function finalizeDmSyncResult(
1061
1068
  messages: XDmMessage[],
1062
1069
  checkpoint: XCheckpoint,
1063
1070
  metadata: Record<string, unknown>,
1064
- ): SyncResult {
1071
+ ): XSyncPage {
1065
1072
  const seenIds = new Set<string>();
1066
1073
  const deduped = messages.filter((message) => {
1067
1074
  if (!message.id || !message.text || seenIds.has(message.id)) return false;
@@ -1104,7 +1111,7 @@ export function finalizeSyncResult(
1104
1111
  checkpoint: XCheckpoint,
1105
1112
  metadata: Record<string, unknown>,
1106
1113
  options?: { originType?: string },
1107
- ): SyncResult {
1114
+ ): XSyncPage {
1108
1115
  const seenIds = new Set<string>();
1109
1116
  const deduped = tweets.filter((tweet) => {
1110
1117
  if (!tweet.id || !tweet.text || seenIds.has(tweet.id)) return false;
@@ -1151,7 +1158,7 @@ export function finalizeLikedTweetsResult(
1151
1158
  pagesRead: number;
1152
1159
  historicalItemsRead: number;
1153
1160
  },
1154
- ): SyncResult {
1161
+ ): XSyncPage {
1155
1162
  // A browser backfill intentionally re-observes the newest page on every run.
1156
1163
  // Do not drop the old boundary: deterministic origin ids let ingestion update
1157
1164
  // metrics/media without creating a second canonical event.
@@ -1498,7 +1505,7 @@ function shouldFallbackToExtension(error: unknown): boolean {
1498
1505
  return isOAuthScopeOrAuthError(error) || isOAuthLookupFallbackError(error);
1499
1506
  }
1500
1507
 
1501
- async function syncWithOAuthFallback<T extends SyncResult>(
1508
+ async function syncWithOAuthFallback<T extends XSyncPage>(
1502
1509
  oauthFn: () => Promise<T>,
1503
1510
  extensionFn: () => Promise<T>,
1504
1511
  ): Promise<T> {
@@ -1511,7 +1518,7 @@ async function syncWithOAuthFallback<T extends SyncResult>(
1511
1518
  }
1512
1519
 
1513
1520
  /** Browser-first feeds fall back to the extension unless `use_oauth` is set. */
1514
- async function syncOAuthWithOptionalFallback<T extends SyncResult>(
1521
+ async function syncOAuthWithOptionalFallback<T extends XSyncPage>(
1515
1522
  config: Record<string, unknown>,
1516
1523
  oauthFn: () => Promise<T>,
1517
1524
  extensionFn: () => Promise<T>,
@@ -1774,7 +1781,7 @@ async function syncViaOAuthApi(
1774
1781
  ctx: SyncContext,
1775
1782
  config: Record<string, unknown>,
1776
1783
  checkpoint: XCheckpoint,
1777
- ): Promise<SyncResult> {
1784
+ ): Promise<XSyncPage> {
1778
1785
  const accessToken = ctx.credentials?.accessToken;
1779
1786
  if (!accessToken) {
1780
1787
  throw new Error("OAuth access token missing for X connector");
@@ -1876,7 +1883,7 @@ async function syncViaExtension(args: {
1876
1883
  /** Extra metadata to fold into the result (e.g. which timeline tab). */
1877
1884
  metadata?: Record<string, unknown>;
1878
1885
  originType?: string;
1879
- }): Promise<SyncResult> {
1886
+ }): Promise<XSyncPage> {
1880
1887
  const { ctx, url, interceptPatterns, parseResponse, maxScrolls, checkpoint } =
1881
1888
  args;
1882
1889
  const result = await extensionNetworkSync<XTweet>({
@@ -1909,7 +1916,7 @@ async function syncSearchViaExtension(
1909
1916
  ctx: SyncContext,
1910
1917
  config: Record<string, unknown>,
1911
1918
  checkpoint: XCheckpoint,
1912
- ): Promise<SyncResult> {
1919
+ ): Promise<XSyncPage> {
1913
1920
  const searchQuery = buildSearchQuery(config);
1914
1921
  const maxScrolls = readMaxPages(config);
1915
1922
  const searchFilter = (config.search_filter as string) ?? "live";
@@ -1930,7 +1937,7 @@ async function syncMyTweetsViaOAuthApi(
1930
1937
  ctx: SyncContext,
1931
1938
  config: Record<string, unknown>,
1932
1939
  checkpoint: XCheckpoint,
1933
- ): Promise<SyncResult> {
1940
+ ): Promise<XSyncPage> {
1934
1941
  const accessToken = ctx.credentials?.accessToken;
1935
1942
  if (!accessToken) {
1936
1943
  throw new Error("OAuth access token missing for my_tweets feed");
@@ -1977,7 +1984,7 @@ async function syncMyTweetsViaExtension(
1977
1984
  ctx: SyncContext,
1978
1985
  config: Record<string, unknown>,
1979
1986
  checkpoint: XCheckpoint,
1980
- ): Promise<SyncResult> {
1987
+ ): Promise<XSyncPage> {
1981
1988
  const accountHandle = await resolveAccountHandle(config);
1982
1989
  const maxScrolls = readMaxPages(config);
1983
1990
  const profileUrl = `https://x.com/${encodeURIComponent(accountHandle)}`;
@@ -1997,7 +2004,7 @@ async function syncLikedTweetsViaExtension(
1997
2004
  ctx: SyncContext,
1998
2005
  config: Record<string, unknown>,
1999
2006
  checkpoint: XCheckpoint,
2000
- ): Promise<SyncResult> {
2007
+ ): Promise<XSyncPage> {
2001
2008
  const accountHandle = await resolveAccountHandle(config);
2002
2009
  const likesUrl = `https://x.com/${encodeURIComponent(accountHandle)}/likes`;
2003
2010
  const dispatcher = requireExtensionDispatcher(ctx);
@@ -2130,7 +2137,7 @@ async function syncLikedTweetsViaOAuthApi(
2130
2137
  ctx: SyncContext,
2131
2138
  config: Record<string, unknown>,
2132
2139
  checkpoint: XCheckpoint,
2133
- ): Promise<SyncResult> {
2140
+ ): Promise<XSyncPage> {
2134
2141
  const accessToken = ctx.credentials?.accessToken;
2135
2142
  if (!accessToken) {
2136
2143
  throw new Error("OAuth access token missing for liked_tweets feed");
@@ -2201,7 +2208,7 @@ async function syncBookmarksViaOAuthApi(
2201
2208
  ctx: SyncContext,
2202
2209
  config: Record<string, unknown>,
2203
2210
  checkpoint: XCheckpoint,
2204
- ): Promise<SyncResult> {
2211
+ ): Promise<XSyncPage> {
2205
2212
  const accessToken = ctx.credentials?.accessToken;
2206
2213
  if (!accessToken) {
2207
2214
  throw new Error("OAuth access token missing for bookmarks feed");
@@ -2240,7 +2247,7 @@ async function syncBookmarksViaExtension(
2240
2247
  ctx: SyncContext,
2241
2248
  config: Record<string, unknown>,
2242
2249
  checkpoint: XCheckpoint,
2243
- ): Promise<SyncResult> {
2250
+ ): Promise<XSyncPage> {
2244
2251
  const maxScrolls = readMaxPages(config);
2245
2252
 
2246
2253
  return syncViaExtension({
@@ -2278,7 +2285,7 @@ async function syncDirectMessagesViaOAuthApi(
2278
2285
  ctx: SyncContext,
2279
2286
  config: Record<string, unknown>,
2280
2287
  checkpoint: XCheckpoint,
2281
- ): Promise<SyncResult> {
2288
+ ): Promise<XSyncPage> {
2282
2289
  const accessToken = ctx.credentials?.accessToken;
2283
2290
  if (!accessToken) {
2284
2291
  throw new Error(
@@ -2332,7 +2339,7 @@ async function syncDirectMessagesViaExtension(
2332
2339
  ctx: SyncContext,
2333
2340
  config: Record<string, unknown>,
2334
2341
  checkpoint: XCheckpoint,
2335
- ): Promise<SyncResult> {
2342
+ ): Promise<XSyncPage> {
2336
2343
  const maxScrolls = readMaxPages(config);
2337
2344
  let authUserId =
2338
2345
  typeof config.account_user_id === "string"
@@ -2378,7 +2385,7 @@ async function syncHomeFeedViaDomScrape(
2378
2385
  ctx: SyncContext,
2379
2386
  config: Record<string, unknown>,
2380
2387
  checkpoint: XCheckpoint,
2381
- ): Promise<SyncResult> {
2388
+ ): Promise<XSyncPage> {
2382
2389
  const maxScrolls = readScrollBudget(config, { defaultMax: 10, cap: 30 });
2383
2390
  const { items: rows, loggedIn } = await extensionDomScrape<HomeFeedRow>({
2384
2391
  dispatcher: requireExtensionDispatcher(ctx),
@@ -3343,6 +3350,14 @@ export default class XConnector extends ConnectorRuntime {
3343
3350
  }
3344
3351
 
3345
3352
  private async syncFeed(ctx: SyncContext): Promise<SyncResult> {
3353
+ // Collected whole, then committed once: an OAuth attempt that fails part
3354
+ // way falls back to the extension, and nothing it gathered may land first.
3355
+ const page = await this.collectFeed(ctx);
3356
+ await ctx.commit(page.events, page.checkpoint);
3357
+ return { status: "complete", ...(page.metadata ? { metadata: page.metadata } : {}) };
3358
+ }
3359
+
3360
+ private async collectFeed(ctx: SyncContext): Promise<XSyncPage> {
3346
3361
  const config = ctx.config as Record<string, unknown>;
3347
3362
  const checkpoint = (ctx.checkpoint ?? {}) as XCheckpoint;
3348
3363
  const feedKey = ctx.feedKey ?? "tweets";
@@ -31,6 +31,13 @@ import {
31
31
  type SyncResult,
32
32
  } from '@lobu/connector-sdk';
33
33
 
34
+ /** What one YouTube sync gathers before it is committed. */
35
+ type YouTubeSyncPage = {
36
+ events: EventEnvelope[];
37
+ checkpoint: Record<string, unknown>;
38
+ metadata?: SyncResult['metadata'];
39
+ };
40
+
34
41
  // ---------------------------------------------------------------------------
35
42
  // YouTube API types
36
43
  // ---------------------------------------------------------------------------
@@ -613,6 +620,12 @@ export default class YouTubeConnector extends ConnectorRuntime {
613
620
  // -------------------------------------------------------------------------
614
621
 
615
622
  private async syncFeed(ctx: SyncContext): Promise<SyncResult> {
623
+ const page = await this.collectFeed(ctx);
624
+ await ctx.commit(page.events, page.checkpoint);
625
+ return { status: 'complete', ...(page.metadata ? { metadata: page.metadata } : {}) };
626
+ }
627
+
628
+ private async collectFeed(ctx: SyncContext): Promise<YouTubeSyncPage> {
616
629
  const auth = this.resolveAuth(ctx);
617
630
 
618
631
  switch (ctx.feedKey) {
@@ -800,7 +813,7 @@ export default class YouTubeConnector extends ConnectorRuntime {
800
813
  // Feed: liked_videos
801
814
  // -------------------------------------------------------------------------
802
815
 
803
- private async syncLikedVideos(ctx: SyncContext, auth: YouTubeAuth): Promise<SyncResult> {
816
+ private async syncLikedVideos(ctx: SyncContext, auth: YouTubeAuth): Promise<YouTubeSyncPage> {
804
817
  this.requireOAuth(auth, 'liked_videos');
805
818
  const maxResults = Math.min(Math.max((ctx.config.max_results as number) ?? 500, 1), 5000);
806
819
  const likesPlaylistId = await this.fetchLikesPlaylistId(auth);
@@ -820,7 +833,7 @@ export default class YouTubeConnector extends ConnectorRuntime {
820
833
  // Feed: playlists
821
834
  // -------------------------------------------------------------------------
822
835
 
823
- private async syncPlaylists(ctx: SyncContext, auth: YouTubeAuth): Promise<SyncResult> {
836
+ private async syncPlaylists(ctx: SyncContext, auth: YouTubeAuth): Promise<YouTubeSyncPage> {
824
837
  this.requireOAuth(auth, 'playlists');
825
838
  const maxPlaylists = Math.min(Math.max((ctx.config.max_playlists as number) ?? 100, 1), 500);
826
839
  const includeItems = (ctx.config.include_items as boolean) ?? true;
@@ -894,7 +907,7 @@ export default class YouTubeConnector extends ConnectorRuntime {
894
907
  // Feed: subscriptions
895
908
  // -------------------------------------------------------------------------
896
909
 
897
- private async syncSubscriptions(ctx: SyncContext, auth: YouTubeAuth): Promise<SyncResult> {
910
+ private async syncSubscriptions(ctx: SyncContext, auth: YouTubeAuth): Promise<YouTubeSyncPage> {
898
911
  this.requireOAuth(auth, 'subscriptions');
899
912
  const maxResults = Math.min(Math.max((ctx.config.max_results as number) ?? 1000, 1), 5000);
900
913
  const events: EventEnvelope[] = [];
@@ -964,7 +977,7 @@ export default class YouTubeConnector extends ConnectorRuntime {
964
977
  // Feed: videos (keyword search)
965
978
  // -------------------------------------------------------------------------
966
979
 
967
- private async syncSearchVideos(ctx: SyncContext, auth: YouTubeAuth): Promise<SyncResult> {
980
+ private async syncSearchVideos(ctx: SyncContext, auth: YouTubeAuth): Promise<YouTubeSyncPage> {
968
981
  const searchQuery = ctx.config.search_query as string;
969
982
  if (!searchQuery) {
970
983
  throw new Error('search_query is required.');
@@ -1250,7 +1263,7 @@ export default class YouTubeConnector extends ConnectorRuntime {
1250
1263
  return events;
1251
1264
  }
1252
1265
 
1253
- private buildListCheckpointResult(events: EventEnvelope[]): SyncResult {
1266
+ private buildListCheckpointResult(events: EventEnvelope[]): YouTubeSyncPage {
1254
1267
  events.sort((a, b) => b.occurred_at.getTime() - a.occurred_at.getTime());
1255
1268
  const latest = events.length > 0 ? events[0].occurred_at.toISOString() : undefined;
1256
1269
  return {
@@ -1,2 +1,2 @@
1
- export { createIsolateConnectorCompiler, findBundledConnectorFile, } from "@lobu/connector-worker/compile";
1
+ export { createIsolateConnectorCompiler, createSourceCapture, findBundledConnectorFile, } from "@lobu/connector-worker/compile";
2
2
  //# sourceMappingURL=connector-compiler.d.ts.map
@@ -1,9 +1,84 @@
1
+ var __create = Object.create;
2
+ var __defProp = Object.defineProperty;
3
+ var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
4
+ var __getOwnPropNames = Object.getOwnPropertyNames;
5
+ var __getProtoOf = Object.getPrototypeOf;
6
+ var __hasOwnProp = Object.prototype.hasOwnProperty;
7
+ var __commonJS = (cb, mod) => function __require() {
8
+ try {
9
+ return mod || (0, cb[__getOwnPropNames(cb)[0]])((mod = { exports: {} }).exports, mod), mod.exports;
10
+ } catch (e) {
11
+ throw mod = 0, e;
12
+ }
13
+ };
14
+ var __copyProps = (to, from, except, desc) => {
15
+ if (from && typeof from === "object" || typeof from === "function") {
16
+ for (let key of __getOwnPropNames(from))
17
+ if (!__hasOwnProp.call(to, key) && key !== except)
18
+ __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
19
+ }
20
+ return to;
21
+ };
22
+ var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
23
+ // If the importer is in node compatibility mode or this is not an ESM
24
+ // file that has been converted to a CommonJS file using a Babel-
25
+ // compatible transform (i.e. "__esModule" has not been set), then set
26
+ // "default" to the CommonJS "module.exports" for node compatibility.
27
+ isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
28
+ mod
29
+ ));
30
+
31
+ // packages/core/dist/contracts/tools/source-files.js
32
+ var require_source_files = __commonJS({
33
+ "packages/core/dist/contracts/tools/source-files.js"(exports) {
34
+ "use strict";
35
+ Object.defineProperty(exports, "__esModule", { value: true });
36
+ exports.SOURCE_MAX_FILES = exports.SOURCE_MAX_BYTES = void 0;
37
+ exports.assertSourcePath = assertSourcePath2;
38
+ exports.validateRetainedSource = validateRetainedSource2;
39
+ exports.SOURCE_MAX_BYTES = 1e6;
40
+ exports.SOURCE_MAX_FILES = 128;
41
+ function assertSourcePath2(path) {
42
+ const parts = path.split("/");
43
+ if (!path || path.length > 240 || /[\\:%]/.test(path) || path.includes("\0") || parts.some((part) => !part || part === "." || part === ".." || part === "node_modules" || part === ".git" || part === ".lobu" || part === ".npmrc" || part.startsWith(".env"))) {
44
+ throw new Error(`Source path must be a portable project file: ${path}`);
45
+ }
46
+ }
47
+ function validateRetainedSource2(sourceCode, sourceFiles, dependencies) {
48
+ assertSourcePath2(sourceFiles.entrypoint);
49
+ const files = Object.entries(sourceFiles.files);
50
+ if (files.length === 0 || files.length > exports.SOURCE_MAX_FILES) {
51
+ throw new Error(`Source must contain between 1 and ${exports.SOURCE_MAX_FILES} files`);
52
+ }
53
+ for (const [path, contents] of files) {
54
+ assertSourcePath2(path);
55
+ if (typeof contents !== "string")
56
+ throw new Error(`Source file must be text: ${path}`);
57
+ }
58
+ if (!Object.keys(sourceFiles.files).includes(sourceFiles.entrypoint) || sourceFiles.files[sourceFiles.entrypoint] !== sourceCode) {
59
+ throw new Error("Source entrypoint must match source_code exactly");
60
+ }
61
+ const packages = Object.entries(dependencies);
62
+ if (packages.length > exports.SOURCE_MAX_FILES)
63
+ throw new Error("Too many source dependencies");
64
+ for (const [name, version] of packages) {
65
+ if (!/^(?:@[a-z0-9._-]+\/)?[a-z0-9._-]+$/.test(name) || !/^\d+\.\d+\.\d+(?:-[0-9A-Za-z.-]+)?(?:\+[0-9A-Za-z.-]+)?$/.test(version)) {
66
+ throw new Error(`Source dependency requires a package name and exact version: ${name}`);
67
+ }
68
+ }
69
+ if (new TextEncoder().encode(JSON.stringify({ sourceFiles, dependencies })).length > exports.SOURCE_MAX_BYTES) {
70
+ throw new Error(`Source files exceed ${exports.SOURCE_MAX_BYTES} bytes`);
71
+ }
72
+ }
73
+ }
74
+ });
75
+
1
76
  // packages/connector-worker/dist/compile/index.js
2
77
  import { existsSync } from "node:fs";
3
78
  import { mkdtemp, readFile, rm, stat, writeFile } from "node:fs/promises";
4
79
  import { createRequire } from "node:module";
5
80
  import { tmpdir } from "node:os";
6
- import { join, resolve } from "node:path";
81
+ import { join as join2, resolve as resolve2 } from "node:path";
7
82
  import { build } from "esbuild";
8
83
 
9
84
  // packages/connector-worker/dist/isolate/eligibility.js
@@ -44,6 +119,103 @@ function computeCompileConfigHash(external) {
44
119
  }
45
120
  var COMPILE_CONFIG_HASH = computeCompileConfigHash(EXTERNAL_RUNTIME_DEPS);
46
121
 
122
+ // packages/connector-worker/dist/compile/source-capture.js
123
+ var import_source_files = __toESM(require_source_files(), 1);
124
+ import { readFileSync, realpathSync } from "node:fs";
125
+ import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
126
+ function within(root, file) {
127
+ const path = relative(root, file);
128
+ return path !== ".." && !path.startsWith(`..${sep}`) && !isAbsolute(path);
129
+ }
130
+ function packageAt(file) {
131
+ let root = dirname(file);
132
+ for (; ; ) {
133
+ try {
134
+ const pkg = JSON.parse(readFileSync(join(root, "package.json"), "utf8"));
135
+ if (typeof pkg.name === "string" && typeof pkg.version === "string") {
136
+ return { root, name: pkg.name, version: pkg.version };
137
+ }
138
+ } catch {
139
+ }
140
+ const parent = dirname(root);
141
+ if (parent === root)
142
+ return null;
143
+ root = parent;
144
+ }
145
+ }
146
+ function createSourceCapture(entry, projectRoot) {
147
+ const root = realpathSync(projectRoot);
148
+ const entrypoint = relative(root, realpathSync(entry)).split(sep).join("/");
149
+ (0, import_source_files.assertSourcePath)(entrypoint);
150
+ const files = /* @__PURE__ */ Object.create(null);
151
+ const dependencies = /* @__PURE__ */ Object.create(null);
152
+ const packageRoots = /* @__PURE__ */ new Set();
153
+ const resolving = /* @__PURE__ */ Symbol("source-capture-resolve");
154
+ const isDependency = (file) => [...packageRoots].some((base) => within(base, file));
155
+ const isProjectFile = (file) => within(root, file) && !isDependency(file);
156
+ const source = () => {
157
+ const sourceFiles = { entrypoint, files: { ...files } };
158
+ (0, import_source_files.validateRetainedSource)(files[entrypoint] ?? "", sourceFiles, dependencies);
159
+ return { sourceFiles, dependencies: { ...dependencies } };
160
+ };
161
+ const plugin = {
162
+ name: "retain-project-source",
163
+ setup(build2) {
164
+ build2.onResolve({ filter: /.*/ }, async (args) => {
165
+ if (args.pluginData?.[resolving])
166
+ return;
167
+ const found = await build2.resolve(args.path, {
168
+ importer: args.importer,
169
+ resolveDir: args.resolveDir,
170
+ kind: args.kind,
171
+ namespace: args.namespace,
172
+ pluginData: { ...args.pluginData, [resolving]: true }
173
+ });
174
+ if (found.pluginData?.[resolving]) {
175
+ found.pluginData = { ...found.pluginData };
176
+ delete found.pluginData[resolving];
177
+ }
178
+ if (found.errors.length || found.external || found.namespace !== "file")
179
+ return found;
180
+ const file = realpathSync(found.path);
181
+ const fromProject = !args.importer || isProjectFile(args.importer);
182
+ const bare = !isAbsolute(args.path) && !args.path.startsWith(".");
183
+ if (bare) {
184
+ const pkg = packageAt(file);
185
+ if (pkg && pkg.root !== root) {
186
+ packageRoots.add(pkg.root);
187
+ if (fromProject) {
188
+ if (dependencies[pkg.name] && dependencies[pkg.name] !== pkg.version) {
189
+ throw new Error(`Conflicting source dependency versions: ${pkg.name}`);
190
+ }
191
+ dependencies[pkg.name] = pkg.version;
192
+ }
193
+ }
194
+ } else if (fromProject && !within(root, file)) {
195
+ throw new Error(`Source import leaves the project: ${args.path}`);
196
+ }
197
+ return found;
198
+ });
199
+ build2.onLoad({ filter: /.*/, namespace: "file" }, (args) => {
200
+ const file = realpathSync(args.path);
201
+ if (isDependency(file))
202
+ return;
203
+ if (!within(root, file))
204
+ throw new Error("Source file leaves the project");
205
+ const path = relative(root, file).split(sep).join("/");
206
+ (0, import_source_files.assertSourcePath)(path);
207
+ const contents = files[path] ?? readFileSync(file, "utf8");
208
+ files[path] = contents;
209
+ if (Buffer.byteLength(JSON.stringify(files), "utf8") > import_source_files.SOURCE_MAX_BYTES) {
210
+ throw new Error(`Source files exceed ${import_source_files.SOURCE_MAX_BYTES} bytes`);
211
+ }
212
+ return { contents, loader: "default", resolveDir: dirname(resolve(file)) };
213
+ });
214
+ }
215
+ };
216
+ return { plugin, source };
217
+ }
218
+
47
219
  // packages/connector-worker/dist/compile/index.js
48
220
  var CONNECTOR_KEY_RE = /^[a-z][a-z0-9]*(?:[._][a-z0-9]+)*$/;
49
221
  function findBundledConnectorFile(key, candidateDirs) {
@@ -55,7 +227,7 @@ function findBundledConnectorFile(key, candidateDirs) {
55
227
  ];
56
228
  for (const dir of candidateDirs) {
57
229
  for (const fileName of candidates) {
58
- const filePath = resolve(dir, fileName);
230
+ const filePath = resolve2(dir, fileName);
59
231
  if (!filePath.startsWith(`${dir}/`))
60
232
  continue;
61
233
  if (existsSync(filePath))
@@ -76,6 +248,10 @@ var ISOLATE_LANE_BUILD_OPTIONS = {
76
248
  supported: { "dynamic-import": false },
77
249
  external: []
78
250
  };
251
+ var RETAINED_SOURCE_BUILD_OPTIONS = {
252
+ minify: true,
253
+ legalComments: "none"
254
+ };
79
255
  function createNpmSpecifierPlugin(options) {
80
256
  return {
81
257
  name: "npm-specifier",
@@ -120,10 +296,10 @@ function resolveSdkFile(specifier) {
120
296
  if (!subpath)
121
297
  return root;
122
298
  const candidates = [
123
- join(rootDir, "dist", `${subpath}.js`),
124
- join(rootDir, "dist", `${subpath}/index.js`),
125
- join(rootDir, "src", `${subpath}.ts`),
126
- join(rootDir, "src", `${subpath}/index.ts`)
299
+ join2(rootDir, "dist", `${subpath}.js`),
300
+ join2(rootDir, "dist", `${subpath}/index.js`),
301
+ join2(rootDir, "src", `${subpath}.ts`),
302
+ join2(rootDir, "src", `${subpath}/index.ts`)
127
303
  ];
128
304
  for (const cand of candidates) {
129
305
  if (existsSync(cand))
@@ -196,12 +372,12 @@ function createIsolateConnectorCompiler(options) {
196
372
  cache.delete(oldest);
197
373
  }
198
374
  }
199
- async function bundleConnectorForIsolate(filePath) {
375
+ async function bundleConnectorForIsolate(filePath, capture) {
200
376
  let mtimeMs = null;
201
377
  try {
202
378
  mtimeMs = (await stat(filePath)).mtimeMs;
203
379
  const cached = cache.get(filePath);
204
- if (cached && cached.mtimeMs === mtimeMs) {
380
+ if (!capture && cached && cached.mtimeMs === mtimeMs) {
205
381
  touch(filePath, cached);
206
382
  return cached.bundle;
207
383
  }
@@ -214,16 +390,17 @@ function createIsolateConnectorCompiler(options) {
214
390
  write: false,
215
391
  metafile: true,
216
392
  minify: false,
393
+ ...capture && RETAINED_SOURCE_BUILD_OPTIONS,
217
394
  sourcemap: false,
218
395
  logLevel: "silent",
219
- plugins
396
+ plugins: capture ? [capture.plugin, ...plugins] : plugins
220
397
  });
221
398
  const code = result.outputFiles[0]?.text ?? "";
222
399
  const bundle = {
223
400
  code,
224
401
  builtins: builtinsFromMetafile(result.metafile)
225
402
  };
226
- if (mtimeMs !== null)
403
+ if (!capture && mtimeMs !== null)
227
404
  touch(filePath, { mtimeMs, bundle });
228
405
  return bundle;
229
406
  }
@@ -233,11 +410,19 @@ function createIsolateConnectorCompiler(options) {
233
410
  throw new IsolateLaneIneligibleError(bundle.builtins, filePath);
234
411
  return bundle.code;
235
412
  }
236
- async function bundleConnectorForIsolateFromSource(sourceCode) {
237
- const tmpDir = await mkdtemp(join(tmpdir(), "lobu-connector-isolate-"));
238
- const sourcePath = join(tmpDir, "source.ts");
413
+ async function compileConnectorArtifactFromFile(filePath, projectRoot) {
414
+ const capture = createSourceCapture(filePath, projectRoot);
415
+ const bundle = await bundleConnectorForIsolate(filePath, capture);
416
+ if (bundle.builtins.length > 0)
417
+ throw new IsolateLaneIneligibleError(bundle.builtins, filePath);
418
+ return { compiledCode: bundle.code, ...capture.source() };
419
+ }
420
+ async function bundleConnectorForIsolateFromSource(sourceCode, retainSource = false, portableOutput = retainSource) {
421
+ const tmpDir = await mkdtemp(join2(tmpdir(), "lobu-connector-isolate-"));
422
+ const sourcePath = join2(tmpDir, "source.ts");
239
423
  try {
240
424
  await writeFile(sourcePath, sourceCode, "utf-8");
425
+ const capture = retainSource ? createSourceCapture(sourcePath, tmpDir) : void 0;
241
426
  const result = await build({
242
427
  ...ISOLATE_LANE_BUILD_OPTIONS,
243
428
  entryPoints: [sourcePath],
@@ -245,33 +430,47 @@ function createIsolateConnectorCompiler(options) {
245
430
  write: false,
246
431
  metafile: true,
247
432
  minify: false,
433
+ ...portableOutput && RETAINED_SOURCE_BUILD_OPTIONS,
434
+ // Uploaded artifacts already have portable identifiers. Renaming them
435
+ // again changes their hash on every normalization pass.
436
+ ...portableOutput && !retainSource && { minify: false, minifyWhitespace: true, minifySyntax: true },
248
437
  sourcemap: false,
249
438
  logLevel: "silent",
250
- nodePaths: [resolve(process.cwd(), "node_modules")],
251
- plugins
439
+ nodePaths: [resolve2(process.cwd(), "node_modules")],
440
+ plugins: [...capture ? [capture.plugin] : [], ...plugins]
252
441
  });
253
442
  const code = result.outputFiles[0]?.text ?? "";
254
443
  return {
255
444
  code,
445
+ ...capture?.source() ?? {},
256
446
  builtins: builtinsFromMetafile(result.metafile)
257
447
  };
258
448
  } finally {
259
449
  await rm(tmpDir, { recursive: true, force: true });
260
450
  }
261
451
  }
262
- async function compileConnectorForIsolateFromSource(sourceCode) {
263
- const bundle = await bundleConnectorForIsolateFromSource(sourceCode);
452
+ async function compileConnectorArtifactFromSource(sourceCode) {
453
+ const bundle = await bundleConnectorForIsolateFromSource(sourceCode, true);
454
+ if (bundle.builtins.length > 0)
455
+ throw new IsolateLaneIneligibleError(bundle.builtins, "<source>");
456
+ return { compiledCode: bundle.code, sourceFiles: bundle.sourceFiles, dependencies: bundle.dependencies };
457
+ }
458
+ async function compileConnectorForIsolateFromSource(sourceCode, portableOutput = false) {
459
+ const bundle = await bundleConnectorForIsolateFromSource(sourceCode, false, portableOutput);
264
460
  if (bundle.builtins.length > 0)
265
461
  throw new IsolateLaneIneligibleError(bundle.builtins, "<source>");
266
462
  return bundle.code;
267
463
  }
268
464
  return {
269
465
  bundleConnectorForIsolate,
466
+ compileConnectorArtifactFromFile,
467
+ compileConnectorArtifactFromSource,
270
468
  compileConnectorForIsolateFromFile,
271
469
  compileConnectorForIsolateFromSource
272
470
  };
273
471
  }
274
472
  export {
275
473
  createIsolateConnectorCompiler,
474
+ createSourceCapture,
276
475
  findBundledConnectorFile
277
476
  };
@@ -5,7 +5,7 @@
5
5
  "server": "dist/server.bundle.mjs",
6
6
  "embeddingsServer": "dist/embeddings-server.mjs"
7
7
  },
8
- "version": "20.2.0",
8
+ "version": "21.0.0",
9
9
  "verify": "dist/verify.mjs"
10
10
  },
11
11
  "device": {
@@ -16,7 +16,7 @@
16
16
  "connector": "node_modules/@lobu/cli/dist/commands/connector.js",
17
17
  "worker": "node_modules/@lobu/connector-worker/dist/bin.js"
18
18
  },
19
- "version": "20.2.0",
19
+ "version": "21.0.0",
20
20
  "verify": "dist/verify.mjs"
21
21
  },
22
22
  "postgres": {
@@ -24,7 +24,7 @@
24
24
  "entries": {
25
25
  "postgres": "dist/index.mjs"
26
26
  },
27
- "version": "20.2.0",
27
+ "version": "21.0.0",
28
28
  "verify": "dist/verify.mjs"
29
29
  },
30
30
  "embeddings": {
@@ -32,7 +32,7 @@
32
32
  "entries": {
33
33
  "embeddings": "dist/embeddings.js"
34
34
  },
35
- "version": "20.2.0",
35
+ "version": "21.0.0",
36
36
  "verify": "dist/verify.mjs"
37
37
  }
38
38
  }