@usesocial/cli 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +14 -0
- package/README.md +1 -1
- package/dist/index.d.mts +2 -1
- package/dist/index.mjs +1839 -1021
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -3,7 +3,6 @@ import { createRequire } from "node:module";
|
|
|
3
3
|
import * as fs from "node:fs";
|
|
4
4
|
import { createReadStream, mkdirSync, readFileSync, realpathSync, writeFileSync } from "node:fs";
|
|
5
5
|
import { fileURLToPath } from "node:url";
|
|
6
|
-
import { createInterface } from "node:readline/promises";
|
|
7
6
|
import { homedir } from "node:os";
|
|
8
7
|
import { dirname, join, resolve } from "node:path";
|
|
9
8
|
import { parseArgs, stripVTControlCharacters, styleText } from "node:util";
|
|
@@ -12,7 +11,7 @@ import { createHash } from "node:crypto";
|
|
|
12
11
|
import { chmod, mkdir, readFile, rm, writeFile } from "node:fs/promises";
|
|
13
12
|
import { dirname as dirname$1, isAbsolute, posix, relative, sep } from "path";
|
|
14
13
|
import * as b from "node:readline";
|
|
15
|
-
import E, { createInterface
|
|
14
|
+
import E, { createInterface } from "node:readline";
|
|
16
15
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
17
16
|
import V, { stdin, stdout } from "node:process";
|
|
18
17
|
import { ReadStream } from "node:tty";
|
|
@@ -134,6 +133,9 @@ var CancelledError = class extends Error {
|
|
|
134
133
|
const MIGRATIONS = [{
|
|
135
134
|
"tag": "0000_modern_stephen_strange",
|
|
136
135
|
"sql": "CREATE TABLE `li_connections` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `object` text,\n `url` text,\n `user_id` text,\n `user_object` text,\n `user_type` text,\n `user_display_name` text,\n `user_public_identifier` text,\n `user_profile_url` text,\n `user_public_picture_url` text,\n `user_description` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `li_connections_created_at_idx` ON `li_connections` (`created_at`);--> statement-breakpoint\nCREATE TABLE `li_conversations` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `last_message_timestamp` integer,\n `muted_until` integer,\n `creator_id` text,\n `headline` text,\n `image_url` text,\n `name` text,\n `object` text,\n `provider` text,\n `type` text,\n `user_id` text,\n `is_1to1` integer,\n `is_archived` integer,\n `is_channel` integer,\n `is_group` integer,\n `is_pinned` integer,\n `is_readonly` integer,\n `unread_count` integer,\n `user_object` text,\n `user_type` text,\n `user_display_name` text,\n `user_public_identifier` text,\n `user_profile_url` text,\n `user_public_picture_url` text,\n `user_description` text,\n `user_specifics_network_distance` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `li_conversations_created_at_idx` ON `li_conversations` (`created_at`);--> statement-breakpoint\nCREATE INDEX `li_conversations_last_message_timestamp_idx` ON `li_conversations` (`last_message_timestamp`);--> statement-breakpoint\nCREATE TABLE `li_messages` (\n `id` text PRIMARY KEY NOT NULL,\n `timestamp` integer,\n `chat_id` text,\n `object` text,\n `provider` text,\n `text` text,\n `sender_id` text,\n `is_deleted` integer,\n `is_delivered` integer,\n `is_edited` integer,\n `is_event` integer,\n `is_hidden` integer,\n `is_mentionned` integer,\n `is_pinned` integer,\n `is_seen` integer,\n `is_sender` integer,\n `sender_object` text,\n `sender_type` text,\n `sender_display_name` text,\n `sender_public_identifier` text,\n `sender_profile_url` text,\n `sender_public_picture_url` text,\n `sender_description` text,\n `sender_specifics_network_distance` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `li_messages_timestamp_idx` ON `li_messages` (`timestamp`);--> statement-breakpoint\nCREATE INDEX `li_messages_chat_id_idx` ON `li_messages` (`chat_id`);--> statement-breakpoint\nCREATE TABLE `li_posts` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `object` text,\n `share_url` text,\n `text` text,\n `url` text,\n `is_repost` integer,\n `user_reacted` text,\n `comments_counter` integer,\n `reposts_counter` integer,\n `analytics_impressions_counter` integer,\n `author_id` text,\n `author_object` text,\n `author_type` text,\n `author_display_name` text,\n `author_public_identifier` text,\n `author_profile_url` text,\n `author_public_picture_url` text,\n `author_description` text,\n `author_specifics_network_distance` text,\n `permissions_can_post_comments` integer,\n `permissions_can_react` integer,\n `permissions_can_share` integer,\n `poll_id` text,\n `poll_question` text,\n `poll_is_open` integer,\n `poll_total_votes_count` integer,\n `quoted_post_id` text,\n `quoted_post_object` text,\n `quoted_post_created_at` integer,\n `quoted_post_text` text,\n `quoted_post_share_url` text,\n `quoted_post_user_reacted` text,\n `quoted_post_comments_counter` integer,\n `quoted_post_reposts_counter` integer,\n `quoted_post_permissions_can_post_comments` integer,\n `quoted_post_permissions_can_react` integer,\n `quoted_post_permissions_can_share` integer,\n `quoted_post_author_id` text,\n `quoted_post_author_object` text,\n `quoted_post_author_type` text,\n `quoted_post_author_display_name` text,\n `quoted_post_author_public_identifier` text,\n `quoted_post_author_profile_url` text,\n `quoted_post_author_public_picture_url` text,\n `quoted_post_author_description` text,\n `quoted_post_author_specifics_network_distance` text,\n `reposted_by_id` text,\n `reposted_by_object` text,\n `reposted_by_type` text,\n `reposted_by_display_name` text,\n `reposted_by_public_identifier` text,\n `reposted_by_profile_url` text,\n `reposted_by_public_picture_url` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `li_posts_created_at_idx` ON `li_posts` (`created_at`);--> statement-breakpoint\nCREATE TABLE `li_requests` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `object` text,\n `type` text,\n `user_id` text,\n `user_object` text,\n `user_type` text,\n `user_display_name` text,\n `user_public_identifier` text,\n `user_profile_url` text,\n `user_public_picture_url` text,\n `user_description` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `li_requests_created_at_idx` ON `li_requests` (`created_at`);--> statement-breakpoint\nCREATE TABLE `sync_state` (\n `collection` text PRIMARY KEY NOT NULL,\n `cursor` text,\n `last_synced` integer,\n `object_count` integer\n);\n--> statement-breakpoint\nCREATE TABLE `x_bookmarks` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `author_id` text,\n `conversation_id` text,\n `in_reply_to_user_id` text,\n `lang` text,\n `reply_settings` text,\n `text` text,\n `url` text,\n `paid_partnership` integer,\n `possibly_sensitive` integer,\n `edit_controls_editable_until` integer,\n `edit_controls_edits_remaining` integer,\n `edit_controls_is_edit_eligible` integer,\n `like_count` integer,\n `retweet_count` integer,\n `reply_count` integer,\n `quote_count` integer,\n `bookmark_count` integer,\n `impression_count` integer,\n `article_cover_media` text,\n `article_plain_text` text,\n `article_preview_text` text,\n `article_title` text,\n `note_tweet_text` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `x_bookmarks_created_at_idx` ON `x_bookmarks` (`created_at`);--> statement-breakpoint\nCREATE TABLE `x_followers` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `username` text,\n `name` text,\n `description` text,\n `location` text,\n `url` text,\n `is_identity_verified` integer,\n `protected` integer,\n `verified` integer,\n `verified_type` text,\n `subscription_type` text,\n `parody` integer,\n `most_recent_tweet_id` text,\n `pinned_tweet_id` text,\n `profile_banner_url` text,\n `profile_image_url` text,\n `affiliation_badge_url` text,\n `affiliation_description` text,\n `affiliation_url` text,\n `subscription_subscribes_to_you` integer,\n `followers_count` integer,\n `following_count` integer,\n `tweet_count` integer,\n `listed_count` integer,\n `like_count` integer,\n `media_count` integer,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `x_followers_created_at_idx` ON `x_followers` (`created_at`);--> statement-breakpoint\nCREATE TABLE `x_following` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `username` text,\n `name` text,\n `description` text,\n `location` text,\n `url` text,\n `is_identity_verified` integer,\n `protected` integer,\n `verified` integer,\n `verified_type` text,\n `subscription_type` text,\n `parody` integer,\n `most_recent_tweet_id` text,\n `pinned_tweet_id` text,\n `profile_banner_url` text,\n `profile_image_url` text,\n `affiliation_badge_url` text,\n `affiliation_description` text,\n `affiliation_url` text,\n `subscription_subscribes_to_you` integer,\n `followers_count` integer,\n `following_count` integer,\n `tweet_count` integer,\n `listed_count` integer,\n `like_count` integer,\n `media_count` integer,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `x_following_created_at_idx` ON `x_following` (`created_at`);--> statement-breakpoint\nCREATE TABLE `x_tweets` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `author_id` text,\n `conversation_id` text,\n `in_reply_to_user_id` text,\n `lang` text,\n `reply_settings` text,\n `text` text,\n `url` text,\n `paid_partnership` integer,\n `possibly_sensitive` integer,\n `edit_controls_editable_until` integer,\n `edit_controls_edits_remaining` integer,\n `edit_controls_is_edit_eligible` integer,\n `like_count` integer,\n `retweet_count` integer,\n `reply_count` integer,\n `quote_count` integer,\n `bookmark_count` integer,\n `impression_count` integer,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `x_tweets_created_at_idx` ON `x_tweets` (`created_at`);"
|
|
136
|
+
}, {
|
|
137
|
+
"tag": "0001_far_beast",
|
|
138
|
+
"sql": "CREATE TABLE `li_profiles` (\n `provider_id` text PRIMARY KEY NOT NULL,\n `handle` text,\n `display_name` text NOT NULL,\n `avatar_url` text,\n `headline` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `li_profiles_handle_idx` ON `li_profiles` (`handle`);--> statement-breakpoint\nCREATE TABLE `x_messages` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `dm_conversation_id` text,\n `event_type` text,\n `sender_id` text,\n `text` text,\n `participant_ids` text,\n `attachments` text,\n `entities` text,\n `referenced_tweets` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `x_messages_created_at_idx` ON `x_messages` (`created_at`);--> statement-breakpoint\nCREATE INDEX `x_messages_dm_conversation_id_idx` ON `x_messages` (`dm_conversation_id`);--> statement-breakpoint\nCREATE TABLE `x_profiles` (\n `provider_id` text PRIMARY KEY NOT NULL,\n `handle` text,\n `display_name` text NOT NULL,\n `avatar_url` text,\n `headline` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `x_profiles_handle_idx` ON `x_profiles` (`handle`);"
|
|
137
139
|
}];
|
|
138
140
|
//#endregion
|
|
139
141
|
//#region ../../packages/sync-engine/src/migrate.ts
|
|
@@ -178,9 +180,7 @@ const invokeWithParams = (fn, params) => {
|
|
|
178
180
|
return normalizedParams ? fn(normalizedParams) : fn();
|
|
179
181
|
};
|
|
180
182
|
const wrapStatement = (statement) => ({
|
|
181
|
-
run: (params) =>
|
|
182
|
-
invokeWithParams(statement.run.bind(statement), params);
|
|
183
|
-
},
|
|
183
|
+
run: (params) => invokeWithParams(statement.run.bind(statement), params),
|
|
184
184
|
get: (params) => invokeWithParams(statement.get.bind(statement), params),
|
|
185
185
|
all: (params) => invokeWithParams(statement.all.bind(statement), params)
|
|
186
186
|
});
|
|
@@ -225,13 +225,17 @@ const cacheTables = new Set([
|
|
|
225
225
|
"x_followers",
|
|
226
226
|
"x_following",
|
|
227
227
|
"x_bookmarks",
|
|
228
|
+
"x_profiles",
|
|
229
|
+
"x_messages",
|
|
228
230
|
"li_conversations",
|
|
229
231
|
"li_messages",
|
|
230
232
|
"li_connections",
|
|
231
233
|
"li_posts",
|
|
234
|
+
"li_profiles",
|
|
232
235
|
"li_requests",
|
|
233
236
|
"sync_state"
|
|
234
237
|
]);
|
|
238
|
+
const profileTables = new Set(["x_profiles", "li_profiles"]);
|
|
235
239
|
const sanitizeAccountId = (accountId) => accountId.replaceAll(/[^A-Za-z0-9._-]/g, "_");
|
|
236
240
|
const quoteIdentifier = (identifier) => `\`${identifier.replaceAll("`", "``")}\``;
|
|
237
241
|
const resolveCachePath = (path) => path === ":memory:" ? path : resolve(path);
|
|
@@ -240,6 +244,10 @@ const numberOrNull = (value) => typeof value === "number" ? value : null;
|
|
|
240
244
|
const assertKnownTable = (table) => {
|
|
241
245
|
if (!cacheTables.has(table)) throw new Error(`Unknown cache table: ${table}`);
|
|
242
246
|
};
|
|
247
|
+
const assertKnownProfileTable = (table) => {
|
|
248
|
+
assertKnownTable(table);
|
|
249
|
+
if (!profileTables.has(table)) throw new Error(`Unknown profile table: ${table}`);
|
|
250
|
+
};
|
|
243
251
|
const cachePath = (args) => join(homedir(), ".social", "cache", `${args.platform}-${sanitizeAccountId(args.accountId)}.db`);
|
|
244
252
|
const openCacheAt = (path, opts) => {
|
|
245
253
|
const dbPath = resolveCachePath(path);
|
|
@@ -268,6 +276,46 @@ const createCache = (path, db) => ({
|
|
|
268
276
|
});
|
|
269
277
|
});
|
|
270
278
|
},
|
|
279
|
+
upsertProfiles: (table, rows) => {
|
|
280
|
+
if (rows.length === 0) return;
|
|
281
|
+
assertKnownProfileTable(table);
|
|
282
|
+
const rowsByProviderId = /* @__PURE__ */ new Map();
|
|
283
|
+
for (const row of rows) rowsByProviderId.set(row.providerId, row);
|
|
284
|
+
const columns = [
|
|
285
|
+
"provider_id",
|
|
286
|
+
"handle",
|
|
287
|
+
"display_name",
|
|
288
|
+
"avatar_url",
|
|
289
|
+
"headline",
|
|
290
|
+
"raw",
|
|
291
|
+
"synced_at"
|
|
292
|
+
];
|
|
293
|
+
const quotedTable = quoteIdentifier(table);
|
|
294
|
+
const nullableColumns = [
|
|
295
|
+
"handle",
|
|
296
|
+
"avatar_url",
|
|
297
|
+
"headline"
|
|
298
|
+
];
|
|
299
|
+
const lastWriteColumns = [
|
|
300
|
+
"display_name",
|
|
301
|
+
"raw",
|
|
302
|
+
"synced_at"
|
|
303
|
+
];
|
|
304
|
+
const updateAssignments = [...nullableColumns.map((column) => `${quoteIdentifier(column)}=COALESCE(excluded.${quoteIdentifier(column)}, ${quotedTable}.${quoteIdentifier(column)})`), ...lastWriteColumns.map((column) => `${quoteIdentifier(column)}=excluded.${quoteIdentifier(column)}`)].join(", ");
|
|
305
|
+
const statement = db.prepare(`INSERT INTO ${quotedTable} (${columns.map(quoteIdentifier).join(", ")}) VALUES (${columns.map((column) => `:${column}`).join(", ")}) ON CONFLICT(provider_id) DO UPDATE SET ${updateAssignments}`);
|
|
306
|
+
const syncedAt = Date.now();
|
|
307
|
+
db.transaction(() => {
|
|
308
|
+
for (const row of rowsByProviderId.values()) statement.run({
|
|
309
|
+
provider_id: row.providerId,
|
|
310
|
+
handle: row.handle,
|
|
311
|
+
display_name: row.displayName,
|
|
312
|
+
avatar_url: row.avatarURL,
|
|
313
|
+
headline: row.headline,
|
|
314
|
+
raw: JSON.stringify(row.raw),
|
|
315
|
+
synced_at: syncedAt
|
|
316
|
+
});
|
|
317
|
+
});
|
|
318
|
+
},
|
|
271
319
|
getCursor: (collection) => {
|
|
272
320
|
const row = db.prepare("SELECT cursor, last_synced, object_count FROM sync_state WHERE collection = :collection").get({ collection });
|
|
273
321
|
if (!row) return;
|
|
@@ -285,6 +333,22 @@ const createCache = (path, db) => ({
|
|
|
285
333
|
object_count: objectCount
|
|
286
334
|
});
|
|
287
335
|
},
|
|
336
|
+
reset: ({ tables, collections }) => {
|
|
337
|
+
for (const table of tables) assertKnownTable(table);
|
|
338
|
+
const results = [];
|
|
339
|
+
db.transaction(() => {
|
|
340
|
+
for (const table of tables) {
|
|
341
|
+
const result = db.prepare(`DELETE FROM ${quoteIdentifier(table)}`).run();
|
|
342
|
+
results.push({
|
|
343
|
+
table,
|
|
344
|
+
rowsDeleted: result?.changes ?? 0
|
|
345
|
+
});
|
|
346
|
+
}
|
|
347
|
+
const deleteState = db.prepare("DELETE FROM sync_state WHERE collection = :collection");
|
|
348
|
+
for (const collection of collections) deleteState.run({ collection });
|
|
349
|
+
});
|
|
350
|
+
return results;
|
|
351
|
+
},
|
|
288
352
|
query: (sql) => db.prepare(sql).all(),
|
|
289
353
|
describe: () => {
|
|
290
354
|
return {
|
|
@@ -535,6 +599,7 @@ const LINKEDIN_COLLECTIONS = [
|
|
|
535
599
|
key: "li_connections",
|
|
536
600
|
platform: "linkedin",
|
|
537
601
|
requests: [{ path: "/users/connections" }],
|
|
602
|
+
requiresFullInitialSync: true,
|
|
538
603
|
pagination: linkedinCursorPagination,
|
|
539
604
|
pageSize: 100,
|
|
540
605
|
estCostBPS: 10,
|
|
@@ -580,6 +645,9 @@ const xPagination = {
|
|
|
580
645
|
cursorPath: "meta.next_token",
|
|
581
646
|
itemsPath: "data"
|
|
582
647
|
};
|
|
648
|
+
const defaultDMEventFields = "attachments,created_at,dm_conversation_id,entities,event_type,id,participant_ids,referenced_tweets,sender_id,text";
|
|
649
|
+
const defaultDMExpansions = "attachments.media_keys,participant_ids,referenced_tweets.id,sender_id";
|
|
650
|
+
const defaultDMUserFields = "affiliation,connection_status,created_at,description,entities,id,location,most_recent_tweet_id,name,pinned_tweet_id,profile_banner_url,profile_image_url,protected,public_metrics,receives_your_dm,subscription_type,url,username,verified,verified_type,withheld";
|
|
583
651
|
const asRecord = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
584
652
|
const read = (value) => value === null || value === void 0 ? null : value;
|
|
585
653
|
const isoToEpochMs = (value) => {
|
|
@@ -592,6 +660,7 @@ const toBool = (value) => {
|
|
|
592
660
|
if (typeof value !== "boolean") return null;
|
|
593
661
|
return value ? 1 : 0;
|
|
594
662
|
};
|
|
663
|
+
const stringifyJSON$1 = (value) => value === null || value === void 0 ? null : JSON.stringify(value);
|
|
595
664
|
const liftTweetColumns = (item) => {
|
|
596
665
|
const editControls = asRecord(item.edit_controls);
|
|
597
666
|
const publicMetrics = asRecord(item.public_metrics);
|
|
@@ -635,6 +704,19 @@ const liftBookmark = (item) => {
|
|
|
635
704
|
raw: JSON.stringify(item)
|
|
636
705
|
};
|
|
637
706
|
};
|
|
707
|
+
const liftDMEvent = (item) => ({
|
|
708
|
+
id: read(item.id),
|
|
709
|
+
created_at: isoToEpochMs(item.created_at),
|
|
710
|
+
dm_conversation_id: read(item.dm_conversation_id),
|
|
711
|
+
event_type: read(item.event_type),
|
|
712
|
+
sender_id: read(item.sender_id),
|
|
713
|
+
text: read(item.text),
|
|
714
|
+
participant_ids: stringifyJSON$1(item.participant_ids),
|
|
715
|
+
attachments: stringifyJSON$1(item.attachments),
|
|
716
|
+
entities: stringifyJSON$1(item.entities),
|
|
717
|
+
referenced_tweets: stringifyJSON$1(item.referenced_tweets),
|
|
718
|
+
raw: JSON.stringify(item)
|
|
719
|
+
});
|
|
638
720
|
const liftXUser = (item) => {
|
|
639
721
|
const affiliation = asRecord(item.affiliation);
|
|
640
722
|
const publicMetrics = asRecord(item.public_metrics);
|
|
@@ -690,6 +772,7 @@ const COLLECTIONS = [...[
|
|
|
690
772
|
platform: "x",
|
|
691
773
|
requests: [{ path: "/users/:id/followers" }],
|
|
692
774
|
ownIdToken: ":id",
|
|
775
|
+
requiresFullInitialSync: true,
|
|
693
776
|
pagination: xPagination,
|
|
694
777
|
pageSize: 1e3,
|
|
695
778
|
estCostBPS: 10,
|
|
@@ -701,6 +784,7 @@ const COLLECTIONS = [...[
|
|
|
701
784
|
platform: "x",
|
|
702
785
|
requests: [{ path: "/users/:id/following" }],
|
|
703
786
|
ownIdToken: ":id",
|
|
787
|
+
requiresFullInitialSync: true,
|
|
704
788
|
pagination: xPagination,
|
|
705
789
|
pageSize: 1e3,
|
|
706
790
|
estCostBPS: 10,
|
|
@@ -717,10 +801,37 @@ const COLLECTIONS = [...[
|
|
|
717
801
|
estCostBPS: 10,
|
|
718
802
|
estUnitsPerPage: 100,
|
|
719
803
|
lift: liftBookmark
|
|
804
|
+
},
|
|
805
|
+
{
|
|
806
|
+
key: "x_messages",
|
|
807
|
+
platform: "x",
|
|
808
|
+
requests: [{
|
|
809
|
+
path: "/dm_events",
|
|
810
|
+
bakedQuery: {
|
|
811
|
+
"dm_event.fields": defaultDMEventFields,
|
|
812
|
+
expansions: defaultDMExpansions,
|
|
813
|
+
"user.fields": defaultDMUserFields
|
|
814
|
+
}
|
|
815
|
+
}],
|
|
816
|
+
pagination: xPagination,
|
|
817
|
+
pageSize: 100,
|
|
818
|
+
estCostBPS: 100,
|
|
819
|
+
estUnitsPerPage: 100,
|
|
820
|
+
sinceField: "created_at",
|
|
821
|
+
lift: liftDMEvent
|
|
720
822
|
}
|
|
721
823
|
], ...LINKEDIN_COLLECTIONS];
|
|
722
824
|
const collectionByKey = (key) => COLLECTIONS.find((collection) => collection.key === key);
|
|
723
825
|
const collectionsFor = (platform) => COLLECTIONS.filter((collection) => collection.platform === platform);
|
|
826
|
+
const syncableCollectionsFor = (platform) => {
|
|
827
|
+
const parentKeys = new Set(COLLECTIONS.map((collection) => collection.parentCollection).filter(Boolean));
|
|
828
|
+
return collectionsFor(platform).filter((collection) => !parentKeys.has(collection.key));
|
|
829
|
+
};
|
|
830
|
+
const resetTablesFor = (collection) => {
|
|
831
|
+
if (!collection.parentCollection) return [collection];
|
|
832
|
+
const parent = collectionByKey(collection.parentCollection);
|
|
833
|
+
return parent ? [collection, parent] : [collection];
|
|
834
|
+
};
|
|
724
835
|
//#endregion
|
|
725
836
|
//#region ../../node_modules/.bun/zod@4.4.3/node_modules/zod/v4/core/core.js
|
|
726
837
|
var _a$2;
|
|
@@ -6973,6 +7084,14 @@ function object(shape, params) {
|
|
|
6973
7084
|
...normalizeParams(params)
|
|
6974
7085
|
});
|
|
6975
7086
|
}
|
|
7087
|
+
function looseObject(shape, params) {
|
|
7088
|
+
return new ZodObject({
|
|
7089
|
+
type: "object",
|
|
7090
|
+
shape,
|
|
7091
|
+
catchall: unknown(),
|
|
7092
|
+
...normalizeParams(params)
|
|
7093
|
+
});
|
|
7094
|
+
}
|
|
6976
7095
|
const ZodUnion = /* @__PURE__ */ $constructor("ZodUnion", (inst, def) => {
|
|
6977
7096
|
$ZodUnion.init(inst, def);
|
|
6978
7097
|
ZodType.init(inst, def);
|
|
@@ -9755,6 +9874,35 @@ pgTable("cli_session_grants", {
|
|
|
9755
9874
|
capabilities: text$1("capabilities").array().notNull(),
|
|
9756
9875
|
createdAt: timestamptz("created_at").notNull().defaultNow()
|
|
9757
9876
|
}, (table) => [index$1("cli_session_grants_user_idx").on(table.userId, table.createdAt.desc()), check("cli_session_grants_scope_alias_check", sql`${table.scopeAlias} in ('read', 'read,write')`)]);
|
|
9877
|
+
pgTable("wrapped_runs", {
|
|
9878
|
+
id: text$1("id").primaryKey().$defaultFn(() => crypto.randomUUID()),
|
|
9879
|
+
userId: text$1("user_id").notNull().references(() => user.id, { onDelete: "cascade" }),
|
|
9880
|
+
connectedAccountId: text$1("connected_account_id").references(() => connectedAccounts.id, { onDelete: "set null" }),
|
|
9881
|
+
platform: text$1("platform", { enum: ["x"] }).notNull().default("x"),
|
|
9882
|
+
handle: text$1("handle").notNull(),
|
|
9883
|
+
platformProfileId: text$1("platform_profile_id"),
|
|
9884
|
+
status: text$1("status", { enum: [
|
|
9885
|
+
"pending",
|
|
9886
|
+
"running",
|
|
9887
|
+
"done",
|
|
9888
|
+
"failed"
|
|
9889
|
+
] }).notNull().default("pending"),
|
|
9890
|
+
depthPct: integer$1("depth_pct").notNull().default(100),
|
|
9891
|
+
followerCount: integer$1("follower_count").notNull().default(0),
|
|
9892
|
+
quotedCredits: integer$1("quoted_credits").notNull().default(0),
|
|
9893
|
+
billedCredits: integer$1("billed_credits").notNull().default(0),
|
|
9894
|
+
snapshot: jsonb("snapshot").$type(),
|
|
9895
|
+
shareSlug: text$1("share_slug").notNull(),
|
|
9896
|
+
error: text$1("error"),
|
|
9897
|
+
createdAt: timestamptz("created_at").notNull().defaultNow(),
|
|
9898
|
+
updatedAt: timestamptz("updated_at").notNull().defaultNow()
|
|
9899
|
+
}, (table) => [
|
|
9900
|
+
uniqueIndex("wrapped_runs_share_slug_key").on(table.shareSlug),
|
|
9901
|
+
index$1("wrapped_runs_user_created_idx").on(table.userId, table.createdAt.desc()),
|
|
9902
|
+
index$1("wrapped_runs_handle_done_idx").on(table.handle, table.createdAt.desc()).where(sql`${table.status} = 'done'`),
|
|
9903
|
+
check("wrapped_runs_platform_check", sql`${table.platform} in ('x')`),
|
|
9904
|
+
check("wrapped_runs_status_check", sql`${table.status} in ('pending', 'running', 'done', 'failed')`)
|
|
9905
|
+
]);
|
|
9758
9906
|
//#endregion
|
|
9759
9907
|
//#region ../../packages/billing/src/pricing.ts
|
|
9760
9908
|
const createPricing = (resources) => {
|
|
@@ -9771,7 +9919,6 @@ const createPricing = (resources) => {
|
|
|
9771
9919
|
//#endregion
|
|
9772
9920
|
//#region ../../packages/sync-engine/src/cost.ts
|
|
9773
9921
|
const creditsToUSD = (credits) => credits / 1e3;
|
|
9774
|
-
const usdToCredits = (usd) => Math.round(usd * 1e3);
|
|
9775
9922
|
const estimatePageCredits = (estCostBPS, estUnitsPerPage) => creditsForBPS(billedBPSFor(estCostBPS * estUnitsPerPage));
|
|
9776
9923
|
const creditsFromHeader = (headers) => {
|
|
9777
9924
|
const rawCredits = headers?.get("social-x-credits-used");
|
|
@@ -9804,176 +9951,580 @@ const createSpendTracker = (limitCredits) => {
|
|
|
9804
9951
|
};
|
|
9805
9952
|
};
|
|
9806
9953
|
//#endregion
|
|
9807
|
-
//#region ../../
|
|
9808
|
-
|
|
9809
|
-
|
|
9810
|
-
|
|
9811
|
-
|
|
9812
|
-
|
|
9813
|
-
|
|
9814
|
-
|
|
9815
|
-
|
|
9816
|
-
|
|
9817
|
-
|
|
9818
|
-
|
|
9819
|
-
|
|
9820
|
-
|
|
9821
|
-
|
|
9822
|
-
|
|
9823
|
-
|
|
9824
|
-
|
|
9954
|
+
//#region ../../packages/sync-engine/src/gate.ts
|
|
9955
|
+
const AUTO_CAP_CREDITS = 5e3;
|
|
9956
|
+
const planSync = (args) => {
|
|
9957
|
+
const cap = args.autoCapCredits ?? 5e3;
|
|
9958
|
+
const firstSync = args.firstSync ?? false;
|
|
9959
|
+
if (args.creditsArg != null) return {
|
|
9960
|
+
mode: "run",
|
|
9961
|
+
billingLimitCredits: args.creditsArg,
|
|
9962
|
+
capAndGo: false
|
|
9963
|
+
};
|
|
9964
|
+
if (!firstSync) return {
|
|
9965
|
+
mode: "run",
|
|
9966
|
+
billingLimitCredits: cap,
|
|
9967
|
+
capAndGo: true
|
|
9968
|
+
};
|
|
9969
|
+
if (args.count != null) {
|
|
9970
|
+
const estCredits = estimateSync({
|
|
9971
|
+
estCostBPS: args.collection.estCostBPS,
|
|
9972
|
+
estUnitsPerPage: args.collection.estUnitsPerPage,
|
|
9973
|
+
pageSize: args.collection.pageSize,
|
|
9974
|
+
totalCount: args.count,
|
|
9975
|
+
limitCredits: cap
|
|
9976
|
+
}).estCredits ?? 0;
|
|
9977
|
+
return {
|
|
9978
|
+
mode: "quote",
|
|
9979
|
+
estCredits,
|
|
9980
|
+
quoteLine: `Run with --credits ${Math.ceil(estCredits * 1.05)} to sync fully.`
|
|
9825
9981
|
};
|
|
9826
|
-
if (actions) {
|
|
9827
|
-
this._onUpdate = actions.onUpdate;
|
|
9828
|
-
this._onDelete = actions.onDelete;
|
|
9829
|
-
}
|
|
9830
|
-
}
|
|
9831
|
-
onUpdate(action) {
|
|
9832
|
-
this._onUpdate = action;
|
|
9833
|
-
return this;
|
|
9834
|
-
}
|
|
9835
|
-
onDelete(action) {
|
|
9836
|
-
this._onDelete = action;
|
|
9837
|
-
return this;
|
|
9838
|
-
}
|
|
9839
|
-
/** @internal */
|
|
9840
|
-
build(table) {
|
|
9841
|
-
return new ForeignKey(table, this);
|
|
9842
|
-
}
|
|
9843
|
-
};
|
|
9844
|
-
var ForeignKey = class {
|
|
9845
|
-
constructor(table, builder) {
|
|
9846
|
-
this.table = table;
|
|
9847
|
-
this.reference = builder.reference;
|
|
9848
|
-
this.onUpdate = builder._onUpdate;
|
|
9849
|
-
this.onDelete = builder._onDelete;
|
|
9850
|
-
}
|
|
9851
|
-
static [entityKind] = "SQLiteForeignKey";
|
|
9852
|
-
reference;
|
|
9853
|
-
onUpdate;
|
|
9854
|
-
onDelete;
|
|
9855
|
-
getName() {
|
|
9856
|
-
const { name, columns, foreignColumns } = this.reference();
|
|
9857
|
-
const columnNames = columns.map((column) => column.name);
|
|
9858
|
-
const foreignColumnNames = foreignColumns.map((column) => column.name);
|
|
9859
|
-
const chunks = [
|
|
9860
|
-
this.table[TableName],
|
|
9861
|
-
...columnNames,
|
|
9862
|
-
foreignColumns[0].table[TableName],
|
|
9863
|
-
...foreignColumnNames
|
|
9864
|
-
];
|
|
9865
|
-
return name ?? `${chunks.join("_")}_fk`;
|
|
9866
9982
|
}
|
|
9983
|
+
return {
|
|
9984
|
+
mode: "reject",
|
|
9985
|
+
message: "Cannot estimate the full first sync cost safely. Provide --credits only after the user approves a safety cap."
|
|
9986
|
+
};
|
|
9867
9987
|
};
|
|
9868
9988
|
//#endregion
|
|
9869
|
-
//#region ../../
|
|
9870
|
-
|
|
9871
|
-
|
|
9872
|
-
|
|
9873
|
-
|
|
9874
|
-
|
|
9875
|
-
|
|
9876
|
-
|
|
9877
|
-
|
|
9878
|
-
|
|
9879
|
-
|
|
9880
|
-
|
|
9881
|
-
actions
|
|
9882
|
-
});
|
|
9883
|
-
return this;
|
|
9884
|
-
}
|
|
9885
|
-
unique(name) {
|
|
9886
|
-
this.config.isUnique = true;
|
|
9887
|
-
this.config.uniqueName = name;
|
|
9888
|
-
return this;
|
|
9889
|
-
}
|
|
9890
|
-
generatedAlwaysAs(as, config) {
|
|
9891
|
-
this.config.generated = {
|
|
9892
|
-
as,
|
|
9893
|
-
type: "always",
|
|
9894
|
-
mode: config?.mode ?? "virtual"
|
|
9895
|
-
};
|
|
9896
|
-
return this;
|
|
9897
|
-
}
|
|
9898
|
-
/** @internal */
|
|
9899
|
-
buildForeignKeys(column, table) {
|
|
9900
|
-
return this.foreignKeyConfigs.map(({ ref, actions }) => {
|
|
9901
|
-
return ((ref2, actions2) => {
|
|
9902
|
-
const builder = new ForeignKeyBuilder(() => {
|
|
9903
|
-
const foreignColumn = ref2();
|
|
9904
|
-
return {
|
|
9905
|
-
columns: [column],
|
|
9906
|
-
foreignColumns: [foreignColumn]
|
|
9907
|
-
};
|
|
9908
|
-
});
|
|
9909
|
-
if (actions2.onUpdate) builder.onUpdate(actions2.onUpdate);
|
|
9910
|
-
if (actions2.onDelete) builder.onDelete(actions2.onDelete);
|
|
9911
|
-
return builder.build(table);
|
|
9912
|
-
})(ref, actions);
|
|
9913
|
-
});
|
|
9989
|
+
//#region ../../packages/sync-engine/src/profiles.ts
|
|
9990
|
+
const profileTableFor = (platform) => platform === "x" ? "x_profiles" : "li_profiles";
|
|
9991
|
+
const itemsSchema = array(record(string(), unknown()));
|
|
9992
|
+
const cursorSchema = union([string(), _null()]);
|
|
9993
|
+
const totalCountSchema = number().int().nonnegative();
|
|
9994
|
+
const isRecord$6 = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
|
|
9995
|
+
const getByPath = (obj, path) => {
|
|
9996
|
+
if (path === void 0) return;
|
|
9997
|
+
let current = obj;
|
|
9998
|
+
for (const segment of path.split(".")) {
|
|
9999
|
+
if (!(isRecord$6(current) && segment in current)) return;
|
|
10000
|
+
current = current[segment];
|
|
9914
10001
|
}
|
|
10002
|
+
return current;
|
|
9915
10003
|
};
|
|
9916
|
-
|
|
9917
|
-
|
|
9918
|
-
|
|
9919
|
-
|
|
9920
|
-
|
|
9921
|
-
|
|
9922
|
-
|
|
10004
|
+
const validateEnvelope = (json, collection, onProgress) => {
|
|
10005
|
+
const { pagination } = collection;
|
|
10006
|
+
const itemsPath = pagination.style === "none" ? void 0 : pagination.itemsPath;
|
|
10007
|
+
const cursorPath = pagination.style === "none" ? void 0 : pagination.cursorPath;
|
|
10008
|
+
const totalCountPath = pagination.style === "none" ? void 0 : pagination.totalCountPath;
|
|
10009
|
+
const itemsResult = itemsSchema.safeParse(getByPath(json, itemsPath));
|
|
10010
|
+
const rawCursor = getByPath(json, cursorPath);
|
|
10011
|
+
const rawTotalCount = getByPath(json, totalCountPath);
|
|
10012
|
+
const cursorValue = rawCursor === void 0 || rawCursor === "" ? null : rawCursor;
|
|
10013
|
+
const cursorResult = cursorSchema.safeParse(cursorValue);
|
|
10014
|
+
const totalCountResult = totalCountSchema.safeParse(rawTotalCount);
|
|
10015
|
+
if (!itemsResult.success) onProgress?.(`Warning: ${collection.key} returned invalid items at ${itemsPath ?? "<root>"}; treating page as empty.`);
|
|
10016
|
+
return {
|
|
10017
|
+
items: itemsResult.success ? itemsResult.data : [],
|
|
10018
|
+
cursor: cursorResult.success ? cursorResult.data : null,
|
|
10019
|
+
totalCount: totalCountResult.success ? totalCountResult.data : null
|
|
10020
|
+
};
|
|
9923
10021
|
};
|
|
9924
|
-
|
|
9925
|
-
|
|
9926
|
-
|
|
9927
|
-
|
|
9928
|
-
|
|
9929
|
-
super(name, "bigint", "SQLiteBigInt");
|
|
9930
|
-
}
|
|
9931
|
-
/** @internal */
|
|
9932
|
-
build(table) {
|
|
9933
|
-
return new SQLiteBigInt(table, this.config);
|
|
9934
|
-
}
|
|
10022
|
+
const substitutePath = (path, args) => {
|
|
10023
|
+
let resolved = path;
|
|
10024
|
+
if (args.ownIdToken) resolved = resolved.replaceAll(args.ownIdToken, args.ownId);
|
|
10025
|
+
if (args.parentIdToken && args.chatId !== void 0) resolved = resolved.replaceAll(args.parentIdToken, args.chatId);
|
|
10026
|
+
return resolved;
|
|
9935
10027
|
};
|
|
9936
|
-
|
|
9937
|
-
|
|
9938
|
-
|
|
9939
|
-
|
|
9940
|
-
|
|
9941
|
-
|
|
9942
|
-
|
|
9943
|
-
|
|
9944
|
-
const decoder = new TextDecoder();
|
|
9945
|
-
return BigInt(decoder.decode(value));
|
|
9946
|
-
}
|
|
9947
|
-
return BigInt(String.fromCodePoint(...value));
|
|
9948
|
-
}
|
|
9949
|
-
mapToDriverValue(value) {
|
|
9950
|
-
return Buffer.from(value.toString());
|
|
9951
|
-
}
|
|
10028
|
+
const buildQuery = (pagination, pageSize, cursor, offset, bakedQuery) => {
|
|
10029
|
+
const params = new URLSearchParams();
|
|
10030
|
+
if (pagination.style !== "none" && pagination.limitParam) params.set(pagination.limitParam, String(pageSize));
|
|
10031
|
+
if (pagination.style !== "none" && cursor && pagination.cursorParam) params.set(pagination.cursorParam, cursor);
|
|
10032
|
+
if (pagination.style === "offset" && pagination.offsetParam) params.set(pagination.offsetParam, String(offset));
|
|
10033
|
+
for (const [key, value] of Object.entries(bakedQuery ?? {})) params.set(key, value);
|
|
10034
|
+
const query = params.toString();
|
|
10035
|
+
return query ? `?${query}` : "";
|
|
9952
10036
|
};
|
|
9953
|
-
|
|
9954
|
-
|
|
9955
|
-
|
|
9956
|
-
super(name, "json", "SQLiteBlobJson");
|
|
9957
|
-
}
|
|
9958
|
-
/** @internal */
|
|
9959
|
-
build(table) {
|
|
9960
|
-
return new SQLiteBlobJson(table, this.config);
|
|
9961
|
-
}
|
|
10037
|
+
const parentColumnFor = (token) => token?.replace(/^:/, "");
|
|
10038
|
+
const applyBakedQueryOverlay = (row, bakedQuery) => {
|
|
10039
|
+
for (const [key, value] of Object.entries(bakedQuery ?? {})) if (Object.hasOwn(row, key)) row[key] = value;
|
|
9962
10040
|
};
|
|
9963
|
-
|
|
9964
|
-
|
|
9965
|
-
|
|
9966
|
-
|
|
9967
|
-
|
|
9968
|
-
|
|
9969
|
-
|
|
9970
|
-
|
|
9971
|
-
|
|
9972
|
-
|
|
9973
|
-
|
|
9974
|
-
|
|
9975
|
-
|
|
9976
|
-
|
|
10041
|
+
const applyParentOverlay = (row, parentIdToken, parentId) => {
|
|
10042
|
+
const parentColumn = parentColumnFor(parentIdToken);
|
|
10043
|
+
if (parentColumn && parentId !== void 0) row[parentColumn] = parentId;
|
|
10044
|
+
};
|
|
10045
|
+
const liftedTimestamp = (collection, item) => {
|
|
10046
|
+
if (!collection.sinceField) return null;
|
|
10047
|
+
const value = collection.lift(item)[collection.sinceField];
|
|
10048
|
+
return typeof value === "number" ? value : null;
|
|
10049
|
+
};
|
|
10050
|
+
const stepPage = (state, page, opts) => {
|
|
10051
|
+
const newestId = state.firstPage && page.items[0] ? String(page.items[0].id) : state.newestId;
|
|
10052
|
+
const rows = [];
|
|
10053
|
+
const ids = [];
|
|
10054
|
+
for (const item of page.items) {
|
|
10055
|
+
const id = String(item.id);
|
|
10056
|
+
if (state.checkpoint !== null && id === state.checkpoint) return {
|
|
10057
|
+
rows,
|
|
10058
|
+
ids,
|
|
10059
|
+
nextState: {
|
|
10060
|
+
...state,
|
|
10061
|
+
newestId,
|
|
10062
|
+
firstPage: false
|
|
10063
|
+
},
|
|
10064
|
+
stoppedReason: "checkpoint",
|
|
10065
|
+
done: true
|
|
10066
|
+
};
|
|
10067
|
+
if (opts.since != null && opts.collection.sinceField) {
|
|
10068
|
+
const timestamp = liftedTimestamp(opts.collection, item);
|
|
10069
|
+
if (timestamp != null && timestamp < opts.since) return {
|
|
10070
|
+
rows,
|
|
10071
|
+
ids,
|
|
10072
|
+
nextState: {
|
|
10073
|
+
...state,
|
|
10074
|
+
newestId,
|
|
10075
|
+
firstPage: false
|
|
10076
|
+
},
|
|
10077
|
+
stoppedReason: "since",
|
|
10078
|
+
done: true
|
|
10079
|
+
};
|
|
10080
|
+
}
|
|
10081
|
+
const row = opts.collection.lift(item);
|
|
10082
|
+
applyBakedQueryOverlay(row, opts.request.bakedQuery);
|
|
10083
|
+
applyParentOverlay(row, opts.parentIdToken, opts.parentId);
|
|
10084
|
+
rows.push(row);
|
|
10085
|
+
if (opts.collectIds) ids.push(id);
|
|
10086
|
+
}
|
|
10087
|
+
if (opts.collection.pagination.style === "offset") {
|
|
10088
|
+
const nextOffset = state.offset + opts.collection.pageSize;
|
|
10089
|
+
const done = page.items.length === 0 || page.totalCount != null && nextOffset >= page.totalCount || page.totalCount == null && page.items.length < opts.collection.pageSize;
|
|
10090
|
+
return {
|
|
10091
|
+
rows,
|
|
10092
|
+
ids,
|
|
10093
|
+
nextState: {
|
|
10094
|
+
...state,
|
|
10095
|
+
offset: done ? state.offset : nextOffset,
|
|
10096
|
+
newestId,
|
|
10097
|
+
firstPage: false
|
|
10098
|
+
},
|
|
10099
|
+
stoppedReason: "complete",
|
|
10100
|
+
done
|
|
10101
|
+
};
|
|
10102
|
+
}
|
|
10103
|
+
const nextCursor = page.cursor;
|
|
10104
|
+
return {
|
|
10105
|
+
rows,
|
|
10106
|
+
ids,
|
|
10107
|
+
nextState: {
|
|
10108
|
+
...state,
|
|
10109
|
+
cursor: nextCursor || null,
|
|
10110
|
+
newestId,
|
|
10111
|
+
firstPage: false
|
|
10112
|
+
},
|
|
10113
|
+
stoppedReason: "complete",
|
|
10114
|
+
done: !nextCursor
|
|
10115
|
+
};
|
|
10116
|
+
};
|
|
10117
|
+
const estimateFor = (collection, limitCredits, totalCount) => ({
|
|
10118
|
+
collectionKey: collection.key,
|
|
10119
|
+
limitCredits,
|
|
10120
|
+
...estimateSync({
|
|
10121
|
+
estCostBPS: collection.estCostBPS,
|
|
10122
|
+
estUnitsPerPage: collection.estUnitsPerPage,
|
|
10123
|
+
pageSize: collection.pageSize,
|
|
10124
|
+
totalCount,
|
|
10125
|
+
limitCredits
|
|
10126
|
+
})
|
|
10127
|
+
});
|
|
10128
|
+
const mergeStoppedReason = (current, next) => {
|
|
10129
|
+
if (current === "credit-limit" || next === "credit-limit") return "credit-limit";
|
|
10130
|
+
if (current === "since" || next === "since") return "since";
|
|
10131
|
+
if (current === "checkpoint" || next === "checkpoint") return "checkpoint";
|
|
10132
|
+
return "complete";
|
|
10133
|
+
};
|
|
10134
|
+
const refinedEstimateMessage = (collection, limitCredits, totalCount) => {
|
|
10135
|
+
const estimate = estimateFor(collection, limitCredits, totalCount);
|
|
10136
|
+
return `${collection.key}: refined estimate ${estimate.estPages ?? 0} page(s), ${estimate.estCredits ?? 0} credits within ${estimate.limitCredits} credit limit.`;
|
|
10137
|
+
};
|
|
10138
|
+
const walkRequest = async (args) => {
|
|
10139
|
+
const checkpoint = args.useCheckpoint ? args.deps.cache.getCursor(args.collection.key)?.cursor ?? null : null;
|
|
10140
|
+
const ids = [];
|
|
10141
|
+
let state = {
|
|
10142
|
+
checkpoint,
|
|
10143
|
+
cursor: null,
|
|
10144
|
+
offset: 0,
|
|
10145
|
+
newestId: null,
|
|
10146
|
+
firstPage: true
|
|
10147
|
+
};
|
|
10148
|
+
let lastPageCredits = null;
|
|
10149
|
+
let pages = 0;
|
|
10150
|
+
let upserted = 0;
|
|
10151
|
+
let stoppedReason = "complete";
|
|
10152
|
+
while (true) {
|
|
10153
|
+
const predictedNext = state.firstPage ? estimatePageCredits(args.collection.estCostBPS, args.collection.estUnitsPerPage) : lastPageCredits ?? estimatePageCredits(args.collection.estCostBPS, args.collection.estUnitsPerPage);
|
|
10154
|
+
if (args.tracker.wouldExceed(predictedNext)) {
|
|
10155
|
+
stoppedReason = "credit-limit";
|
|
10156
|
+
break;
|
|
10157
|
+
}
|
|
10158
|
+
const path = `${args.deps.platform}` + substitutePath(args.request.path, {
|
|
10159
|
+
ownId: args.deps.ownId,
|
|
10160
|
+
ownIdToken: args.collection.ownIdToken,
|
|
10161
|
+
chatId: args.parentId,
|
|
10162
|
+
parentIdToken: args.parentIdToken
|
|
10163
|
+
}) + buildQuery(args.collection.pagination, args.collection.pageSize, state.cursor, state.offset, args.request.bakedQuery);
|
|
10164
|
+
const response = await args.deps.api.get(path, { headers: args.deps.accountHeaders });
|
|
10165
|
+
const credits = creditsFromHeader(response.headers);
|
|
10166
|
+
args.tracker.add(credits);
|
|
10167
|
+
lastPageCredits = credits;
|
|
10168
|
+
pages++;
|
|
10169
|
+
const json = await response.json();
|
|
10170
|
+
const page = validateEnvelope(json, args.collection, args.onProgress);
|
|
10171
|
+
const extractor = args.deps.profileExtractors?.[`GET ${args.request.path}`];
|
|
10172
|
+
if (extractor) {
|
|
10173
|
+
const profiles = extractor(json);
|
|
10174
|
+
if (profiles.length > 0) args.deps.cache.upsertProfiles(profileTableFor(args.deps.platform), profiles);
|
|
10175
|
+
}
|
|
10176
|
+
if (state.firstPage && page.totalCount != null) args.onProgress?.(refinedEstimateMessage(args.collection, args.billingLimitCredits, page.totalCount));
|
|
10177
|
+
const step = stepPage(state, page, {
|
|
10178
|
+
collection: args.collection,
|
|
10179
|
+
request: args.request,
|
|
10180
|
+
since: args.since,
|
|
10181
|
+
collectIds: args.collectIds,
|
|
10182
|
+
parentId: args.parentId,
|
|
10183
|
+
parentIdToken: args.parentIdToken
|
|
10184
|
+
});
|
|
10185
|
+
args.deps.cache.upsert(args.collection.key, step.rows);
|
|
10186
|
+
upserted += step.rows.length;
|
|
10187
|
+
ids.push(...step.ids);
|
|
10188
|
+
state = step.nextState;
|
|
10189
|
+
await args.onPage?.({
|
|
10190
|
+
collectionKey: args.collection.key,
|
|
10191
|
+
pagesCompleted: pages,
|
|
10192
|
+
objectsUpserted: upserted,
|
|
10193
|
+
creditsSpent: args.tracker.spent()
|
|
10194
|
+
});
|
|
10195
|
+
if (args.expectedTotalCount != null && upserted >= args.expectedTotalCount) {
|
|
10196
|
+
stoppedReason = "complete";
|
|
10197
|
+
break;
|
|
10198
|
+
}
|
|
10199
|
+
if (step.done) {
|
|
10200
|
+
stoppedReason = step.stoppedReason;
|
|
10201
|
+
break;
|
|
10202
|
+
}
|
|
10203
|
+
}
|
|
10204
|
+
return {
|
|
10205
|
+
ids,
|
|
10206
|
+
newestId: state.newestId,
|
|
10207
|
+
pages,
|
|
10208
|
+
upserted,
|
|
10209
|
+
checkpoint,
|
|
10210
|
+
stoppedReason
|
|
10211
|
+
};
|
|
10212
|
+
};
|
|
10213
|
+
const singleRequestCheckpoint = (args) => ({
|
|
10214
|
+
collection: args.collection.key,
|
|
10215
|
+
cursor: args.result.newestId ?? args.result.checkpoint,
|
|
10216
|
+
objectCount: args.result.upserted,
|
|
10217
|
+
lastSynced: Date.now()
|
|
10218
|
+
});
|
|
10219
|
+
const runSync = async (deps, opts) => {
|
|
10220
|
+
const { collection } = opts;
|
|
10221
|
+
if (opts.since != null && collection.sinceField === void 0) throw new Error(`Collection ${collection.key} does not support --since`);
|
|
10222
|
+
if (opts.since != null && collection.requiresFullInitialSync && deps.cache.getCursor(collection.key)?.lastSynced == null) throw new Error(`Collection ${collection.key} requires a full initial sync before --since`);
|
|
10223
|
+
const estimate = estimateFor(collection, opts.billingLimitCredits, opts.expectedTotalCount);
|
|
10224
|
+
if (opts.onEstimate && !await opts.onEstimate(estimate)) return {
|
|
10225
|
+
collection: collection.key,
|
|
10226
|
+
objectsUpserted: 0,
|
|
10227
|
+
pages: 0,
|
|
10228
|
+
creditsSpent: 0,
|
|
10229
|
+
stoppedReason: "aborted",
|
|
10230
|
+
partial: false
|
|
10231
|
+
};
|
|
10232
|
+
const tracker = createSpendTracker(opts.billingLimitCredits);
|
|
10233
|
+
let pages = 0;
|
|
10234
|
+
let upserted = 0;
|
|
10235
|
+
let stoppedReason = "complete";
|
|
10236
|
+
let sawCreditLimit = false;
|
|
10237
|
+
const pendingCursors = [];
|
|
10238
|
+
const addResult = (result) => {
|
|
10239
|
+
pages += result.pages;
|
|
10240
|
+
upserted += result.upserted;
|
|
10241
|
+
stoppedReason = mergeStoppedReason(stoppedReason, result.stoppedReason);
|
|
10242
|
+
sawCreditLimit = sawCreditLimit || result.stoppedReason === "credit-limit";
|
|
10243
|
+
};
|
|
10244
|
+
if (collection.parentCollection) {
|
|
10245
|
+
const parentCollection = collectionByKey(collection.parentCollection);
|
|
10246
|
+
if (!parentCollection) throw new Error(`Missing parent collection ${collection.parentCollection}`);
|
|
10247
|
+
const parentIds = [];
|
|
10248
|
+
const parentPersistsCheckpoint = parentCollection.requests.length === 1;
|
|
10249
|
+
for (const request of parentCollection.requests) {
|
|
10250
|
+
const result = await walkRequest({
|
|
10251
|
+
deps,
|
|
10252
|
+
collection: parentCollection,
|
|
10253
|
+
request,
|
|
10254
|
+
tracker,
|
|
10255
|
+
billingLimitCredits: opts.billingLimitCredits,
|
|
10256
|
+
since: opts.since,
|
|
10257
|
+
useCheckpoint: false,
|
|
10258
|
+
collectIds: true,
|
|
10259
|
+
onPage: opts.onPage,
|
|
10260
|
+
onProgress: opts.onProgress
|
|
10261
|
+
});
|
|
10262
|
+
addResult(result);
|
|
10263
|
+
parentIds.push(...result.ids);
|
|
10264
|
+
if (parentPersistsCheckpoint) pendingCursors.push(singleRequestCheckpoint({
|
|
10265
|
+
collection: parentCollection,
|
|
10266
|
+
result
|
|
10267
|
+
}));
|
|
10268
|
+
if (result.stoppedReason === "credit-limit") break;
|
|
10269
|
+
}
|
|
10270
|
+
if (!parentPersistsCheckpoint) pendingCursors.push({
|
|
10271
|
+
collection: parentCollection.key,
|
|
10272
|
+
cursor: null,
|
|
10273
|
+
objectCount: upserted,
|
|
10274
|
+
lastSynced: Date.now()
|
|
10275
|
+
});
|
|
10276
|
+
if (!sawCreditLimit) for (const parentId of parentIds) {
|
|
10277
|
+
for (const request of collection.requests) {
|
|
10278
|
+
const result = await walkRequest({
|
|
10279
|
+
deps,
|
|
10280
|
+
collection,
|
|
10281
|
+
request,
|
|
10282
|
+
tracker,
|
|
10283
|
+
billingLimitCredits: opts.billingLimitCredits,
|
|
10284
|
+
since: opts.since,
|
|
10285
|
+
useCheckpoint: false,
|
|
10286
|
+
parentId,
|
|
10287
|
+
parentIdToken: collection.parentIdToken,
|
|
10288
|
+
onPage: opts.onPage,
|
|
10289
|
+
onProgress: opts.onProgress
|
|
10290
|
+
});
|
|
10291
|
+
addResult(result);
|
|
10292
|
+
if (result.stoppedReason === "credit-limit") break;
|
|
10293
|
+
}
|
|
10294
|
+
if (sawCreditLimit) break;
|
|
10295
|
+
}
|
|
10296
|
+
pendingCursors.push({
|
|
10297
|
+
collection: collection.key,
|
|
10298
|
+
cursor: null,
|
|
10299
|
+
objectCount: upserted,
|
|
10300
|
+
lastSynced: Date.now()
|
|
10301
|
+
});
|
|
10302
|
+
} else {
|
|
10303
|
+
const useCheckpoint = collection.requests.length === 1;
|
|
10304
|
+
for (const request of collection.requests) {
|
|
10305
|
+
const result = await walkRequest({
|
|
10306
|
+
deps,
|
|
10307
|
+
collection,
|
|
10308
|
+
request,
|
|
10309
|
+
tracker,
|
|
10310
|
+
billingLimitCredits: opts.billingLimitCredits,
|
|
10311
|
+
since: opts.since,
|
|
10312
|
+
expectedTotalCount: opts.expectedTotalCount,
|
|
10313
|
+
useCheckpoint,
|
|
10314
|
+
onPage: opts.onPage,
|
|
10315
|
+
onProgress: opts.onProgress
|
|
10316
|
+
});
|
|
10317
|
+
addResult(result);
|
|
10318
|
+
if (useCheckpoint) pendingCursors.push(singleRequestCheckpoint({
|
|
10319
|
+
collection,
|
|
10320
|
+
result
|
|
10321
|
+
}));
|
|
10322
|
+
if (result.stoppedReason === "credit-limit") break;
|
|
10323
|
+
}
|
|
10324
|
+
if (!useCheckpoint) pendingCursors.push({
|
|
10325
|
+
collection: collection.key,
|
|
10326
|
+
cursor: null,
|
|
10327
|
+
objectCount: upserted,
|
|
10328
|
+
lastSynced: Date.now()
|
|
10329
|
+
});
|
|
10330
|
+
}
|
|
10331
|
+
const creditsSpent = tracker.spent();
|
|
10332
|
+
if (!sawCreditLimit) for (const pending of pendingCursors) deps.cache.setCursor(pending.collection, pending.cursor, pending.objectCount, pending.lastSynced);
|
|
10333
|
+
return {
|
|
10334
|
+
collection: collection.key,
|
|
10335
|
+
objectsUpserted: upserted,
|
|
10336
|
+
pages,
|
|
10337
|
+
creditsSpent,
|
|
10338
|
+
stoppedReason,
|
|
10339
|
+
partial: sawCreditLimit
|
|
10340
|
+
};
|
|
10341
|
+
};
|
|
10342
|
+
const ensureFreshCache = async (deps, opts) => {
|
|
10343
|
+
const now = opts.now ?? Date.now();
|
|
10344
|
+
const cursor = deps.cache.getCursor(opts.collection.key);
|
|
10345
|
+
if (cursor?.lastSynced == null) return { status: "never-synced" };
|
|
10346
|
+
const ttl = (opts.ttlSeconds ?? 900) * 1e3;
|
|
10347
|
+
if (now - cursor.lastSynced <= ttl) return { status: "fresh" };
|
|
10348
|
+
return {
|
|
10349
|
+
status: "refreshed",
|
|
10350
|
+
summary: await runSync(deps, {
|
|
10351
|
+
collection: opts.collection,
|
|
10352
|
+
billingLimitCredits: AUTO_CAP_CREDITS,
|
|
10353
|
+
onEstimate: () => true
|
|
10354
|
+
})
|
|
10355
|
+
};
|
|
10356
|
+
};
|
|
10357
|
+
//#endregion
|
|
10358
|
+
//#region ../../node_modules/.bun/drizzle-orm@0.43.1+ae5d55cf414a93be/node_modules/drizzle-orm/sqlite-core/foreign-keys.js
|
|
10359
|
+
var ForeignKeyBuilder = class {
|
|
10360
|
+
static [entityKind] = "SQLiteForeignKeyBuilder";
|
|
10361
|
+
/** @internal */
|
|
10362
|
+
reference;
|
|
10363
|
+
/** @internal */
|
|
10364
|
+
_onUpdate;
|
|
10365
|
+
/** @internal */
|
|
10366
|
+
_onDelete;
|
|
10367
|
+
constructor(config, actions) {
|
|
10368
|
+
this.reference = () => {
|
|
10369
|
+
const { name, columns, foreignColumns } = config();
|
|
10370
|
+
return {
|
|
10371
|
+
name,
|
|
10372
|
+
columns,
|
|
10373
|
+
foreignTable: foreignColumns[0].table,
|
|
10374
|
+
foreignColumns
|
|
10375
|
+
};
|
|
10376
|
+
};
|
|
10377
|
+
if (actions) {
|
|
10378
|
+
this._onUpdate = actions.onUpdate;
|
|
10379
|
+
this._onDelete = actions.onDelete;
|
|
10380
|
+
}
|
|
10381
|
+
}
|
|
10382
|
+
onUpdate(action) {
|
|
10383
|
+
this._onUpdate = action;
|
|
10384
|
+
return this;
|
|
10385
|
+
}
|
|
10386
|
+
onDelete(action) {
|
|
10387
|
+
this._onDelete = action;
|
|
10388
|
+
return this;
|
|
10389
|
+
}
|
|
10390
|
+
/** @internal */
|
|
10391
|
+
build(table) {
|
|
10392
|
+
return new ForeignKey(table, this);
|
|
10393
|
+
}
|
|
10394
|
+
};
|
|
10395
|
+
var ForeignKey = class {
|
|
10396
|
+
constructor(table, builder) {
|
|
10397
|
+
this.table = table;
|
|
10398
|
+
this.reference = builder.reference;
|
|
10399
|
+
this.onUpdate = builder._onUpdate;
|
|
10400
|
+
this.onDelete = builder._onDelete;
|
|
10401
|
+
}
|
|
10402
|
+
static [entityKind] = "SQLiteForeignKey";
|
|
10403
|
+
reference;
|
|
10404
|
+
onUpdate;
|
|
10405
|
+
onDelete;
|
|
10406
|
+
getName() {
|
|
10407
|
+
const { name, columns, foreignColumns } = this.reference();
|
|
10408
|
+
const columnNames = columns.map((column) => column.name);
|
|
10409
|
+
const foreignColumnNames = foreignColumns.map((column) => column.name);
|
|
10410
|
+
const chunks = [
|
|
10411
|
+
this.table[TableName],
|
|
10412
|
+
...columnNames,
|
|
10413
|
+
foreignColumns[0].table[TableName],
|
|
10414
|
+
...foreignColumnNames
|
|
10415
|
+
];
|
|
10416
|
+
return name ?? `${chunks.join("_")}_fk`;
|
|
10417
|
+
}
|
|
10418
|
+
};
|
|
10419
|
+
//#endregion
|
|
10420
|
+
//#region ../../node_modules/.bun/drizzle-orm@0.43.1+ae5d55cf414a93be/node_modules/drizzle-orm/sqlite-core/unique-constraint.js
|
|
10421
|
+
function uniqueKeyName(table, columns) {
|
|
10422
|
+
return `${table[TableName]}_${columns.join("_")}_unique`;
|
|
10423
|
+
}
|
|
10424
|
+
//#endregion
|
|
10425
|
+
//#region ../../node_modules/.bun/drizzle-orm@0.43.1+ae5d55cf414a93be/node_modules/drizzle-orm/sqlite-core/columns/common.js
|
|
10426
|
+
var SQLiteColumnBuilder = class extends ColumnBuilder {
|
|
10427
|
+
static [entityKind] = "SQLiteColumnBuilder";
|
|
10428
|
+
foreignKeyConfigs = [];
|
|
10429
|
+
references(ref, actions = {}) {
|
|
10430
|
+
this.foreignKeyConfigs.push({
|
|
10431
|
+
ref,
|
|
10432
|
+
actions
|
|
10433
|
+
});
|
|
10434
|
+
return this;
|
|
10435
|
+
}
|
|
10436
|
+
unique(name) {
|
|
10437
|
+
this.config.isUnique = true;
|
|
10438
|
+
this.config.uniqueName = name;
|
|
10439
|
+
return this;
|
|
10440
|
+
}
|
|
10441
|
+
generatedAlwaysAs(as, config) {
|
|
10442
|
+
this.config.generated = {
|
|
10443
|
+
as,
|
|
10444
|
+
type: "always",
|
|
10445
|
+
mode: config?.mode ?? "virtual"
|
|
10446
|
+
};
|
|
10447
|
+
return this;
|
|
10448
|
+
}
|
|
10449
|
+
/** @internal */
|
|
10450
|
+
buildForeignKeys(column, table) {
|
|
10451
|
+
return this.foreignKeyConfigs.map(({ ref, actions }) => {
|
|
10452
|
+
return ((ref2, actions2) => {
|
|
10453
|
+
const builder = new ForeignKeyBuilder(() => {
|
|
10454
|
+
const foreignColumn = ref2();
|
|
10455
|
+
return {
|
|
10456
|
+
columns: [column],
|
|
10457
|
+
foreignColumns: [foreignColumn]
|
|
10458
|
+
};
|
|
10459
|
+
});
|
|
10460
|
+
if (actions2.onUpdate) builder.onUpdate(actions2.onUpdate);
|
|
10461
|
+
if (actions2.onDelete) builder.onDelete(actions2.onDelete);
|
|
10462
|
+
return builder.build(table);
|
|
10463
|
+
})(ref, actions);
|
|
10464
|
+
});
|
|
10465
|
+
}
|
|
10466
|
+
};
|
|
10467
|
+
var SQLiteColumn = class extends Column {
|
|
10468
|
+
constructor(table, config) {
|
|
10469
|
+
if (!config.uniqueName) config.uniqueName = uniqueKeyName(table, [config.name]);
|
|
10470
|
+
super(table, config);
|
|
10471
|
+
this.table = table;
|
|
10472
|
+
}
|
|
10473
|
+
static [entityKind] = "SQLiteColumn";
|
|
10474
|
+
};
|
|
10475
|
+
//#endregion
|
|
10476
|
+
//#region ../../node_modules/.bun/drizzle-orm@0.43.1+ae5d55cf414a93be/node_modules/drizzle-orm/sqlite-core/columns/blob.js
|
|
10477
|
+
var SQLiteBigIntBuilder = class extends SQLiteColumnBuilder {
|
|
10478
|
+
static [entityKind] = "SQLiteBigIntBuilder";
|
|
10479
|
+
constructor(name) {
|
|
10480
|
+
super(name, "bigint", "SQLiteBigInt");
|
|
10481
|
+
}
|
|
10482
|
+
/** @internal */
|
|
10483
|
+
build(table) {
|
|
10484
|
+
return new SQLiteBigInt(table, this.config);
|
|
10485
|
+
}
|
|
10486
|
+
};
|
|
10487
|
+
var SQLiteBigInt = class extends SQLiteColumn {
|
|
10488
|
+
static [entityKind] = "SQLiteBigInt";
|
|
10489
|
+
getSQLType() {
|
|
10490
|
+
return "blob";
|
|
10491
|
+
}
|
|
10492
|
+
mapFromDriverValue(value) {
|
|
10493
|
+
if (Buffer.isBuffer(value)) return BigInt(value.toString());
|
|
10494
|
+
if (value instanceof ArrayBuffer) {
|
|
10495
|
+
const decoder = new TextDecoder();
|
|
10496
|
+
return BigInt(decoder.decode(value));
|
|
10497
|
+
}
|
|
10498
|
+
return BigInt(String.fromCodePoint(...value));
|
|
10499
|
+
}
|
|
10500
|
+
mapToDriverValue(value) {
|
|
10501
|
+
return Buffer.from(value.toString());
|
|
10502
|
+
}
|
|
10503
|
+
};
|
|
10504
|
+
var SQLiteBlobJsonBuilder = class extends SQLiteColumnBuilder {
|
|
10505
|
+
static [entityKind] = "SQLiteBlobJsonBuilder";
|
|
10506
|
+
constructor(name) {
|
|
10507
|
+
super(name, "json", "SQLiteBlobJson");
|
|
10508
|
+
}
|
|
10509
|
+
/** @internal */
|
|
10510
|
+
build(table) {
|
|
10511
|
+
return new SQLiteBlobJson(table, this.config);
|
|
10512
|
+
}
|
|
10513
|
+
};
|
|
10514
|
+
var SQLiteBlobJson = class extends SQLiteColumn {
|
|
10515
|
+
static [entityKind] = "SQLiteBlobJson";
|
|
10516
|
+
getSQLType() {
|
|
10517
|
+
return "blob";
|
|
10518
|
+
}
|
|
10519
|
+
mapFromDriverValue(value) {
|
|
10520
|
+
if (Buffer.isBuffer(value)) return JSON.parse(value.toString());
|
|
10521
|
+
if (value instanceof ArrayBuffer) {
|
|
10522
|
+
const decoder = new TextDecoder();
|
|
10523
|
+
return JSON.parse(decoder.decode(value));
|
|
10524
|
+
}
|
|
10525
|
+
return JSON.parse(String.fromCodePoint(...value));
|
|
10526
|
+
}
|
|
10527
|
+
mapToDriverValue(value) {
|
|
9977
10528
|
return Buffer.from(JSON.stringify(value));
|
|
9978
10529
|
}
|
|
9979
10530
|
};
|
|
@@ -10425,6 +10976,28 @@ sqliteTable("x_bookmarks", {
|
|
|
10425
10976
|
noteTweetText: text("note_tweet_text"),
|
|
10426
10977
|
...rawColumns()
|
|
10427
10978
|
}, (table) => [index("x_bookmarks_created_at_idx").on(table.createdAt)]);
|
|
10979
|
+
const profileColumns = () => ({
|
|
10980
|
+
providerId: text("provider_id").primaryKey(),
|
|
10981
|
+
handle: text("handle"),
|
|
10982
|
+
displayName: text("display_name").notNull(),
|
|
10983
|
+
avatarURL: text("avatar_url"),
|
|
10984
|
+
headline: text("headline"),
|
|
10985
|
+
...rawColumns()
|
|
10986
|
+
});
|
|
10987
|
+
sqliteTable("x_profiles", profileColumns(), (table) => [index("x_profiles_handle_idx").on(table.handle)]);
|
|
10988
|
+
sqliteTable("x_messages", {
|
|
10989
|
+
id: idColumn(),
|
|
10990
|
+
createdAt: integer("created_at"),
|
|
10991
|
+
dmConversationId: text("dm_conversation_id"),
|
|
10992
|
+
eventType: text("event_type"),
|
|
10993
|
+
senderId: text("sender_id"),
|
|
10994
|
+
text: text("text"),
|
|
10995
|
+
participantIds: text("participant_ids"),
|
|
10996
|
+
attachments: text("attachments"),
|
|
10997
|
+
entities: text("entities"),
|
|
10998
|
+
referencedTweets: text("referenced_tweets"),
|
|
10999
|
+
...rawColumns()
|
|
11000
|
+
}, (table) => [index("x_messages_created_at_idx").on(table.createdAt), index("x_messages_dm_conversation_id_idx").on(table.dmConversationId)]);
|
|
10428
11001
|
const xUserColumns = () => ({
|
|
10429
11002
|
id: idColumn(),
|
|
10430
11003
|
createdAt: integer("created_at"),
|
|
@@ -10472,6 +11045,7 @@ sqliteTable("li_connections", {
|
|
|
10472
11045
|
userDescription: text("user_description"),
|
|
10473
11046
|
...rawColumns()
|
|
10474
11047
|
}, (table) => [index("li_connections_created_at_idx").on(table.createdAt)]);
|
|
11048
|
+
sqliteTable("li_profiles", profileColumns(), (table) => [index("li_profiles_handle_idx").on(table.handle)]);
|
|
10475
11049
|
sqliteTable("li_conversations", {
|
|
10476
11050
|
id: idColumn(),
|
|
10477
11051
|
createdAt: integer("created_at"),
|
|
@@ -10538,404 +11112,91 @@ sqliteTable("li_posts", {
|
|
|
10538
11112
|
URL: text("url"),
|
|
10539
11113
|
isRepost: integer("is_repost"),
|
|
10540
11114
|
userReacted: text("user_reacted"),
|
|
10541
|
-
commentsCounter: integer("comments_counter"),
|
|
10542
|
-
repostsCounter: integer("reposts_counter"),
|
|
10543
|
-
analyticsImpressionsCounter: integer("analytics_impressions_counter"),
|
|
10544
|
-
authorId: text("author_id"),
|
|
10545
|
-
authorObject: text("author_object"),
|
|
10546
|
-
authorType: text("author_type"),
|
|
10547
|
-
authorDisplayName: text("author_display_name"),
|
|
10548
|
-
authorPublicIdentifier: text("author_public_identifier"),
|
|
10549
|
-
authorProfileURL: text("author_profile_url"),
|
|
10550
|
-
authorPublicPictureURL: text("author_public_picture_url"),
|
|
10551
|
-
authorDescription: text("author_description"),
|
|
10552
|
-
authorSpecificsNetworkDistance: text("author_specifics_network_distance"),
|
|
10553
|
-
permissionsCanPostComments: integer("permissions_can_post_comments"),
|
|
10554
|
-
permissionsCanReact: integer("permissions_can_react"),
|
|
10555
|
-
permissionsCanShare: integer("permissions_can_share"),
|
|
10556
|
-
pollId: text("poll_id"),
|
|
10557
|
-
pollQuestion: text("poll_question"),
|
|
10558
|
-
pollIsOpen: integer("poll_is_open"),
|
|
10559
|
-
pollTotalVotesCount: integer("poll_total_votes_count"),
|
|
10560
|
-
quotedPostId: text("quoted_post_id"),
|
|
10561
|
-
quotedPostObject: text("quoted_post_object"),
|
|
10562
|
-
quotedPostCreatedAt: integer("quoted_post_created_at"),
|
|
10563
|
-
quotedPostText: text("quoted_post_text"),
|
|
10564
|
-
quotedPostShareURL: text("quoted_post_share_url"),
|
|
10565
|
-
quotedPostUserReacted: text("quoted_post_user_reacted"),
|
|
10566
|
-
quotedPostCommentsCounter: integer("quoted_post_comments_counter"),
|
|
10567
|
-
quotedPostRepostsCounter: integer("quoted_post_reposts_counter"),
|
|
10568
|
-
quotedPostPermissionsCanPostComments: integer("quoted_post_permissions_can_post_comments"),
|
|
10569
|
-
quotedPostPermissionsCanReact: integer("quoted_post_permissions_can_react"),
|
|
10570
|
-
quotedPostPermissionsCanShare: integer("quoted_post_permissions_can_share"),
|
|
10571
|
-
quotedPostAuthorId: text("quoted_post_author_id"),
|
|
10572
|
-
quotedPostAuthorObject: text("quoted_post_author_object"),
|
|
10573
|
-
quotedPostAuthorType: text("quoted_post_author_type"),
|
|
10574
|
-
quotedPostAuthorDisplayName: text("quoted_post_author_display_name"),
|
|
10575
|
-
quotedPostAuthorPublicIdentifier: text("quoted_post_author_public_identifier"),
|
|
10576
|
-
quotedPostAuthorProfileURL: text("quoted_post_author_profile_url"),
|
|
10577
|
-
quotedPostAuthorPublicPictureURL: text("quoted_post_author_public_picture_url"),
|
|
10578
|
-
quotedPostAuthorDescription: text("quoted_post_author_description"),
|
|
10579
|
-
quotedPostAuthorSpecificsNetworkDistance: text("quoted_post_author_specifics_network_distance"),
|
|
10580
|
-
repostedById: text("reposted_by_id"),
|
|
10581
|
-
repostedByObject: text("reposted_by_object"),
|
|
10582
|
-
repostedByType: text("reposted_by_type"),
|
|
10583
|
-
repostedByDisplayName: text("reposted_by_display_name"),
|
|
10584
|
-
repostedByPublicIdentifier: text("reposted_by_public_identifier"),
|
|
10585
|
-
repostedByProfileURL: text("reposted_by_profile_url"),
|
|
10586
|
-
repostedByPublicPictureURL: text("reposted_by_public_picture_url"),
|
|
10587
|
-
...rawColumns()
|
|
10588
|
-
}, (table) => [index("li_posts_created_at_idx").on(table.createdAt)]);
|
|
10589
|
-
sqliteTable("li_requests", {
|
|
10590
|
-
id: idColumn(),
|
|
10591
|
-
createdAt: integer("created_at"),
|
|
10592
|
-
object: text("object"),
|
|
10593
|
-
type: text("type"),
|
|
10594
|
-
userId: text("user_id"),
|
|
10595
|
-
userObject: text("user_object"),
|
|
10596
|
-
userType: text("user_type"),
|
|
10597
|
-
userDisplayName: text("user_display_name"),
|
|
10598
|
-
userPublicIdentifier: text("user_public_identifier"),
|
|
10599
|
-
userProfileURL: text("user_profile_url"),
|
|
10600
|
-
userPublicPictureURL: text("user_public_picture_url"),
|
|
10601
|
-
userDescription: text("user_description"),
|
|
10602
|
-
...rawColumns()
|
|
10603
|
-
}, (table) => [index("li_requests_created_at_idx").on(table.createdAt)]);
|
|
10604
|
-
sqliteTable("sync_state", {
|
|
10605
|
-
collection: text("collection").primaryKey(),
|
|
10606
|
-
cursor: text("cursor"),
|
|
10607
|
-
lastSynced: integer("last_synced"),
|
|
10608
|
-
objectCount: integer("object_count")
|
|
10609
|
-
});
|
|
10610
|
-
//#endregion
|
|
10611
|
-
//#region ../../packages/sync-engine/src/since.ts
|
|
10612
|
-
const durationPattern = /^(\d+):(hours|days|weeks)$/;
|
|
10613
|
-
const isoPattern = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}Z)?$/;
|
|
10614
|
-
const unitMs = {
|
|
10615
|
-
hours: 36e5,
|
|
10616
|
-
days: 864e5,
|
|
10617
|
-
weeks: 6048e5
|
|
10618
|
-
};
|
|
10619
|
-
const parseSince = (input, now = Date.now()) => {
|
|
10620
|
-
const value = input.trim();
|
|
10621
|
-
const duration = durationPattern.exec(value);
|
|
10622
|
-
if (duration != null) return now - Number.parseInt(duration[1] ?? "", 10) * unitMs[duration[2]];
|
|
10623
|
-
const parsed = isoPattern.test(value) ? Date.parse(value) : NaN;
|
|
10624
|
-
if (Number.isNaN(parsed)) throw new Error("Invalid --since value. Use an ISO 8601 date/datetime like 2026-06-01 or 2026-06-01T00:00:00Z, or a compact duration like 2:days, 3:weeks, or 5:hours.");
|
|
10625
|
-
return parsed;
|
|
10626
|
-
};
|
|
10627
|
-
const itemsSchema = array(record(string(), unknown()));
|
|
10628
|
-
const cursorSchema = union([string(), _null()]);
|
|
10629
|
-
const totalCountSchema = number().int().nonnegative();
|
|
10630
|
-
const isRecord$6 = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
|
|
10631
|
-
const getByPath = (obj, path) => {
|
|
10632
|
-
if (path === void 0) return;
|
|
10633
|
-
let current = obj;
|
|
10634
|
-
for (const segment of path.split(".")) {
|
|
10635
|
-
if (!(isRecord$6(current) && segment in current)) return;
|
|
10636
|
-
current = current[segment];
|
|
10637
|
-
}
|
|
10638
|
-
return current;
|
|
10639
|
-
};
|
|
10640
|
-
const validateEnvelope = (json, collection, onProgress) => {
|
|
10641
|
-
const { pagination } = collection;
|
|
10642
|
-
const itemsPath = pagination.style === "none" ? void 0 : pagination.itemsPath;
|
|
10643
|
-
const cursorPath = pagination.style === "none" ? void 0 : pagination.cursorPath;
|
|
10644
|
-
const totalCountPath = pagination.style === "none" ? void 0 : pagination.totalCountPath;
|
|
10645
|
-
const itemsResult = itemsSchema.safeParse(getByPath(json, itemsPath));
|
|
10646
|
-
const rawCursor = getByPath(json, cursorPath);
|
|
10647
|
-
const rawTotalCount = getByPath(json, totalCountPath);
|
|
10648
|
-
const cursorValue = rawCursor === void 0 || rawCursor === "" ? null : rawCursor;
|
|
10649
|
-
const cursorResult = cursorSchema.safeParse(cursorValue);
|
|
10650
|
-
const totalCountResult = totalCountSchema.safeParse(rawTotalCount);
|
|
10651
|
-
if (!itemsResult.success) onProgress?.(`Warning: ${collection.key} returned invalid items at ${itemsPath ?? "<root>"}; treating page as empty.`);
|
|
10652
|
-
return {
|
|
10653
|
-
items: itemsResult.success ? itemsResult.data : [],
|
|
10654
|
-
cursor: cursorResult.success ? cursorResult.data : null,
|
|
10655
|
-
totalCount: totalCountResult.success ? totalCountResult.data : null
|
|
10656
|
-
};
|
|
10657
|
-
};
|
|
10658
|
-
const substitutePath = (path, args) => {
|
|
10659
|
-
let resolved = path;
|
|
10660
|
-
if (args.ownIdToken) resolved = resolved.replaceAll(args.ownIdToken, args.ownId);
|
|
10661
|
-
if (args.parentIdToken && args.chatId !== void 0) resolved = resolved.replaceAll(args.parentIdToken, args.chatId);
|
|
10662
|
-
return resolved;
|
|
10663
|
-
};
|
|
10664
|
-
const buildQuery = (pagination, pageSize, cursor, offset, bakedQuery) => {
|
|
10665
|
-
const params = new URLSearchParams();
|
|
10666
|
-
if (pagination.style !== "none" && pagination.limitParam) params.set(pagination.limitParam, String(pageSize));
|
|
10667
|
-
if (pagination.style !== "none" && cursor && pagination.cursorParam) params.set(pagination.cursorParam, cursor);
|
|
10668
|
-
if (pagination.style === "offset" && pagination.offsetParam) params.set(pagination.offsetParam, String(offset));
|
|
10669
|
-
for (const [key, value] of Object.entries(bakedQuery ?? {})) params.set(key, value);
|
|
10670
|
-
const query = params.toString();
|
|
10671
|
-
return query ? `?${query}` : "";
|
|
10672
|
-
};
|
|
10673
|
-
const parentColumnFor = (token) => token?.replace(/^:/, "");
|
|
10674
|
-
const applyBakedQueryOverlay = (row, bakedQuery) => {
|
|
10675
|
-
for (const [key, value] of Object.entries(bakedQuery ?? {})) if (Object.hasOwn(row, key)) row[key] = value;
|
|
10676
|
-
};
|
|
10677
|
-
const applyParentOverlay = (row, parentIdToken, parentId) => {
|
|
10678
|
-
const parentColumn = parentColumnFor(parentIdToken);
|
|
10679
|
-
if (parentColumn && parentId !== void 0) row[parentColumn] = parentId;
|
|
10680
|
-
};
|
|
10681
|
-
const liftedTimestamp = (collection, item) => {
|
|
10682
|
-
if (!collection.sinceField) return null;
|
|
10683
|
-
const value = collection.lift(item)[collection.sinceField];
|
|
10684
|
-
return typeof value === "number" ? value : null;
|
|
10685
|
-
};
|
|
10686
|
-
const stepPage = (state, page, opts) => {
|
|
10687
|
-
const newestId = state.firstPage && page.items[0] ? String(page.items[0].id) : state.newestId;
|
|
10688
|
-
const rows = [];
|
|
10689
|
-
const ids = [];
|
|
10690
|
-
for (const item of page.items) {
|
|
10691
|
-
const id = String(item.id);
|
|
10692
|
-
if (state.checkpoint !== null && id === state.checkpoint) return {
|
|
10693
|
-
rows,
|
|
10694
|
-
ids,
|
|
10695
|
-
nextState: {
|
|
10696
|
-
...state,
|
|
10697
|
-
newestId,
|
|
10698
|
-
firstPage: false
|
|
10699
|
-
},
|
|
10700
|
-
stoppedReason: "checkpoint",
|
|
10701
|
-
done: true
|
|
10702
|
-
};
|
|
10703
|
-
if (opts.since != null && opts.collection.sinceField) {
|
|
10704
|
-
const timestamp = liftedTimestamp(opts.collection, item);
|
|
10705
|
-
if (timestamp != null && timestamp < opts.since) return {
|
|
10706
|
-
rows,
|
|
10707
|
-
ids,
|
|
10708
|
-
nextState: {
|
|
10709
|
-
...state,
|
|
10710
|
-
newestId,
|
|
10711
|
-
firstPage: false
|
|
10712
|
-
},
|
|
10713
|
-
stoppedReason: "since",
|
|
10714
|
-
done: true
|
|
10715
|
-
};
|
|
10716
|
-
}
|
|
10717
|
-
const row = opts.collection.lift(item);
|
|
10718
|
-
applyBakedQueryOverlay(row, opts.request.bakedQuery);
|
|
10719
|
-
applyParentOverlay(row, opts.parentIdToken, opts.parentId);
|
|
10720
|
-
rows.push(row);
|
|
10721
|
-
if (opts.collectIds) ids.push(id);
|
|
10722
|
-
}
|
|
10723
|
-
if (opts.collection.pagination.style === "offset") {
|
|
10724
|
-
const nextOffset = state.offset + opts.collection.pageSize;
|
|
10725
|
-
const done = page.items.length === 0 || page.totalCount != null && nextOffset >= page.totalCount || page.totalCount == null && page.items.length < opts.collection.pageSize;
|
|
10726
|
-
return {
|
|
10727
|
-
rows,
|
|
10728
|
-
ids,
|
|
10729
|
-
nextState: {
|
|
10730
|
-
...state,
|
|
10731
|
-
offset: done ? state.offset : nextOffset,
|
|
10732
|
-
newestId,
|
|
10733
|
-
firstPage: false
|
|
10734
|
-
},
|
|
10735
|
-
stoppedReason: "complete",
|
|
10736
|
-
done
|
|
10737
|
-
};
|
|
10738
|
-
}
|
|
10739
|
-
const nextCursor = page.cursor;
|
|
10740
|
-
return {
|
|
10741
|
-
rows,
|
|
10742
|
-
ids,
|
|
10743
|
-
nextState: {
|
|
10744
|
-
...state,
|
|
10745
|
-
cursor: nextCursor || null,
|
|
10746
|
-
newestId,
|
|
10747
|
-
firstPage: false
|
|
10748
|
-
},
|
|
10749
|
-
stoppedReason: "complete",
|
|
10750
|
-
done: !nextCursor
|
|
10751
|
-
};
|
|
10752
|
-
};
|
|
10753
|
-
const estimateFor = (collection, limitCredits, totalCount) => ({
|
|
10754
|
-
collectionKey: collection.key,
|
|
10755
|
-
...estimateSync({
|
|
10756
|
-
estCostBPS: collection.estCostBPS,
|
|
10757
|
-
estUnitsPerPage: collection.estUnitsPerPage,
|
|
10758
|
-
pageSize: collection.pageSize,
|
|
10759
|
-
totalCount,
|
|
10760
|
-
limitCredits
|
|
10761
|
-
})
|
|
11115
|
+
commentsCounter: integer("comments_counter"),
|
|
11116
|
+
repostsCounter: integer("reposts_counter"),
|
|
11117
|
+
analyticsImpressionsCounter: integer("analytics_impressions_counter"),
|
|
11118
|
+
authorId: text("author_id"),
|
|
11119
|
+
authorObject: text("author_object"),
|
|
11120
|
+
authorType: text("author_type"),
|
|
11121
|
+
authorDisplayName: text("author_display_name"),
|
|
11122
|
+
authorPublicIdentifier: text("author_public_identifier"),
|
|
11123
|
+
authorProfileURL: text("author_profile_url"),
|
|
11124
|
+
authorPublicPictureURL: text("author_public_picture_url"),
|
|
11125
|
+
authorDescription: text("author_description"),
|
|
11126
|
+
authorSpecificsNetworkDistance: text("author_specifics_network_distance"),
|
|
11127
|
+
permissionsCanPostComments: integer("permissions_can_post_comments"),
|
|
11128
|
+
permissionsCanReact: integer("permissions_can_react"),
|
|
11129
|
+
permissionsCanShare: integer("permissions_can_share"),
|
|
11130
|
+
pollId: text("poll_id"),
|
|
11131
|
+
pollQuestion: text("poll_question"),
|
|
11132
|
+
pollIsOpen: integer("poll_is_open"),
|
|
11133
|
+
pollTotalVotesCount: integer("poll_total_votes_count"),
|
|
11134
|
+
quotedPostId: text("quoted_post_id"),
|
|
11135
|
+
quotedPostObject: text("quoted_post_object"),
|
|
11136
|
+
quotedPostCreatedAt: integer("quoted_post_created_at"),
|
|
11137
|
+
quotedPostText: text("quoted_post_text"),
|
|
11138
|
+
quotedPostShareURL: text("quoted_post_share_url"),
|
|
11139
|
+
quotedPostUserReacted: text("quoted_post_user_reacted"),
|
|
11140
|
+
quotedPostCommentsCounter: integer("quoted_post_comments_counter"),
|
|
11141
|
+
quotedPostRepostsCounter: integer("quoted_post_reposts_counter"),
|
|
11142
|
+
quotedPostPermissionsCanPostComments: integer("quoted_post_permissions_can_post_comments"),
|
|
11143
|
+
quotedPostPermissionsCanReact: integer("quoted_post_permissions_can_react"),
|
|
11144
|
+
quotedPostPermissionsCanShare: integer("quoted_post_permissions_can_share"),
|
|
11145
|
+
quotedPostAuthorId: text("quoted_post_author_id"),
|
|
11146
|
+
quotedPostAuthorObject: text("quoted_post_author_object"),
|
|
11147
|
+
quotedPostAuthorType: text("quoted_post_author_type"),
|
|
11148
|
+
quotedPostAuthorDisplayName: text("quoted_post_author_display_name"),
|
|
11149
|
+
quotedPostAuthorPublicIdentifier: text("quoted_post_author_public_identifier"),
|
|
11150
|
+
quotedPostAuthorProfileURL: text("quoted_post_author_profile_url"),
|
|
11151
|
+
quotedPostAuthorPublicPictureURL: text("quoted_post_author_public_picture_url"),
|
|
11152
|
+
quotedPostAuthorDescription: text("quoted_post_author_description"),
|
|
11153
|
+
quotedPostAuthorSpecificsNetworkDistance: text("quoted_post_author_specifics_network_distance"),
|
|
11154
|
+
repostedById: text("reposted_by_id"),
|
|
11155
|
+
repostedByObject: text("reposted_by_object"),
|
|
11156
|
+
repostedByType: text("reposted_by_type"),
|
|
11157
|
+
repostedByDisplayName: text("reposted_by_display_name"),
|
|
11158
|
+
repostedByPublicIdentifier: text("reposted_by_public_identifier"),
|
|
11159
|
+
repostedByProfileURL: text("reposted_by_profile_url"),
|
|
11160
|
+
repostedByPublicPictureURL: text("reposted_by_public_picture_url"),
|
|
11161
|
+
...rawColumns()
|
|
11162
|
+
}, (table) => [index("li_posts_created_at_idx").on(table.createdAt)]);
|
|
11163
|
+
sqliteTable("li_requests", {
|
|
11164
|
+
id: idColumn(),
|
|
11165
|
+
createdAt: integer("created_at"),
|
|
11166
|
+
object: text("object"),
|
|
11167
|
+
type: text("type"),
|
|
11168
|
+
userId: text("user_id"),
|
|
11169
|
+
userObject: text("user_object"),
|
|
11170
|
+
userType: text("user_type"),
|
|
11171
|
+
userDisplayName: text("user_display_name"),
|
|
11172
|
+
userPublicIdentifier: text("user_public_identifier"),
|
|
11173
|
+
userProfileURL: text("user_profile_url"),
|
|
11174
|
+
userPublicPictureURL: text("user_public_picture_url"),
|
|
11175
|
+
userDescription: text("user_description"),
|
|
11176
|
+
...rawColumns()
|
|
11177
|
+
}, (table) => [index("li_requests_created_at_idx").on(table.createdAt)]);
|
|
11178
|
+
sqliteTable("sync_state", {
|
|
11179
|
+
collection: text("collection").primaryKey(),
|
|
11180
|
+
cursor: text("cursor"),
|
|
11181
|
+
lastSynced: integer("last_synced"),
|
|
11182
|
+
objectCount: integer("object_count")
|
|
10762
11183
|
});
|
|
10763
|
-
|
|
10764
|
-
|
|
10765
|
-
|
|
10766
|
-
|
|
10767
|
-
|
|
10768
|
-
|
|
10769
|
-
|
|
10770
|
-
|
|
10771
|
-
return `${collection.key}: refined estimate ${estimate.estPages ?? 0} page(s), ${estimate.estCredits ?? 0} credits ($${(estimate.estUSD ?? 0).toFixed(2)}) within $${estimate.limitUSD.toFixed(2)} limit.`;
|
|
10772
|
-
};
|
|
10773
|
-
const walkRequest = async (args) => {
|
|
10774
|
-
const checkpoint = args.useCheckpoint ? args.deps.cache.getCursor(args.collection.key)?.cursor ?? null : null;
|
|
10775
|
-
const ids = [];
|
|
10776
|
-
let state = {
|
|
10777
|
-
checkpoint,
|
|
10778
|
-
cursor: null,
|
|
10779
|
-
offset: 0,
|
|
10780
|
-
newestId: null,
|
|
10781
|
-
firstPage: true
|
|
10782
|
-
};
|
|
10783
|
-
let lastPageCredits = null;
|
|
10784
|
-
let pages = 0;
|
|
10785
|
-
let upserted = 0;
|
|
10786
|
-
let stoppedReason = "complete";
|
|
10787
|
-
while (true) {
|
|
10788
|
-
const predictedNext = state.firstPage ? estimatePageCredits(args.collection.estCostBPS, args.collection.estUnitsPerPage) : lastPageCredits ?? estimatePageCredits(args.collection.estCostBPS, args.collection.estUnitsPerPage);
|
|
10789
|
-
if (args.tracker.wouldExceed(predictedNext)) {
|
|
10790
|
-
stoppedReason = "billing-limit";
|
|
10791
|
-
break;
|
|
10792
|
-
}
|
|
10793
|
-
const path = `${args.deps.platform}` + substitutePath(args.request.path, {
|
|
10794
|
-
ownId: args.deps.ownId,
|
|
10795
|
-
ownIdToken: args.collection.ownIdToken,
|
|
10796
|
-
chatId: args.parentId,
|
|
10797
|
-
parentIdToken: args.parentIdToken
|
|
10798
|
-
}) + buildQuery(args.collection.pagination, args.collection.pageSize, state.cursor, state.offset, args.request.bakedQuery);
|
|
10799
|
-
const response = await args.deps.api.get(path, { headers: args.deps.accountHeaders });
|
|
10800
|
-
const credits = creditsFromHeader(response.headers);
|
|
10801
|
-
args.tracker.add(credits);
|
|
10802
|
-
lastPageCredits = credits;
|
|
10803
|
-
pages++;
|
|
10804
|
-
const page = validateEnvelope(await response.json(), args.collection, args.onProgress);
|
|
10805
|
-
if (state.firstPage && page.totalCount != null) args.onProgress?.(refinedEstimateMessage(args.collection, args.billingLimitCredits, page.totalCount));
|
|
10806
|
-
const step = stepPage(state, page, {
|
|
10807
|
-
collection: args.collection,
|
|
10808
|
-
request: args.request,
|
|
10809
|
-
since: args.since,
|
|
10810
|
-
collectIds: args.collectIds,
|
|
10811
|
-
parentId: args.parentId,
|
|
10812
|
-
parentIdToken: args.parentIdToken
|
|
10813
|
-
});
|
|
10814
|
-
args.deps.cache.upsert(args.collection.key, step.rows);
|
|
10815
|
-
upserted += step.rows.length;
|
|
10816
|
-
ids.push(...step.ids);
|
|
10817
|
-
state = step.nextState;
|
|
10818
|
-
if (step.done) {
|
|
10819
|
-
stoppedReason = step.stoppedReason;
|
|
10820
|
-
break;
|
|
10821
|
-
}
|
|
10822
|
-
}
|
|
10823
|
-
return {
|
|
10824
|
-
ids,
|
|
10825
|
-
newestId: state.newestId,
|
|
10826
|
-
pages,
|
|
10827
|
-
upserted,
|
|
10828
|
-
checkpoint,
|
|
10829
|
-
stoppedReason
|
|
10830
|
-
};
|
|
10831
|
-
};
|
|
10832
|
-
const persistSingleRequestCheckpoint = (args) => {
|
|
10833
|
-
args.cache.setCursor(args.collection.key, args.result.newestId ?? args.result.checkpoint, args.result.upserted, Date.now());
|
|
11184
|
+
//#endregion
|
|
11185
|
+
//#region ../../packages/sync-engine/src/since.ts
|
|
11186
|
+
const durationPattern = /^(\d+):(hours|days|weeks)$/;
|
|
11187
|
+
const isoPattern = /^\d{4}-\d{2}-\d{2}(?:T\d{2}:\d{2}:\d{2}Z)?$/;
|
|
11188
|
+
const unitMs = {
|
|
11189
|
+
hours: 36e5,
|
|
11190
|
+
days: 864e5,
|
|
11191
|
+
weeks: 6048e5
|
|
10834
11192
|
};
|
|
10835
|
-
const
|
|
10836
|
-
const
|
|
10837
|
-
|
|
10838
|
-
|
|
10839
|
-
|
|
10840
|
-
|
|
10841
|
-
|
|
10842
|
-
pages: 0,
|
|
10843
|
-
creditsSpent: 0,
|
|
10844
|
-
usdSpent: 0,
|
|
10845
|
-
stoppedReason: "aborted",
|
|
10846
|
-
partial: false
|
|
10847
|
-
};
|
|
10848
|
-
const tracker = createSpendTracker(opts.billingLimitCredits);
|
|
10849
|
-
let pages = 0;
|
|
10850
|
-
let upserted = 0;
|
|
10851
|
-
let stoppedReason = "complete";
|
|
10852
|
-
let sawBillingLimit = false;
|
|
10853
|
-
const addResult = (result) => {
|
|
10854
|
-
pages += result.pages;
|
|
10855
|
-
upserted += result.upserted;
|
|
10856
|
-
stoppedReason = mergeStoppedReason(stoppedReason, result.stoppedReason);
|
|
10857
|
-
sawBillingLimit = sawBillingLimit || result.stoppedReason === "billing-limit";
|
|
10858
|
-
};
|
|
10859
|
-
if (collection.parentCollection) {
|
|
10860
|
-
const parentCollection = collectionByKey(collection.parentCollection);
|
|
10861
|
-
if (!parentCollection) throw new Error(`Missing parent collection ${collection.parentCollection}`);
|
|
10862
|
-
const parentIds = [];
|
|
10863
|
-
const parentPersistsCheckpoint = parentCollection.requests.length === 1;
|
|
10864
|
-
for (const request of parentCollection.requests) {
|
|
10865
|
-
const result = await walkRequest({
|
|
10866
|
-
deps,
|
|
10867
|
-
collection: parentCollection,
|
|
10868
|
-
request,
|
|
10869
|
-
tracker,
|
|
10870
|
-
billingLimitCredits: opts.billingLimitCredits,
|
|
10871
|
-
since: opts.since,
|
|
10872
|
-
useCheckpoint: false,
|
|
10873
|
-
collectIds: true,
|
|
10874
|
-
onProgress: opts.onProgress
|
|
10875
|
-
});
|
|
10876
|
-
addResult(result);
|
|
10877
|
-
parentIds.push(...result.ids);
|
|
10878
|
-
if (parentPersistsCheckpoint) persistSingleRequestCheckpoint({
|
|
10879
|
-
cache: deps.cache,
|
|
10880
|
-
collection: parentCollection,
|
|
10881
|
-
result
|
|
10882
|
-
});
|
|
10883
|
-
if (result.stoppedReason === "billing-limit") break;
|
|
10884
|
-
}
|
|
10885
|
-
if (!parentPersistsCheckpoint) deps.cache.setCursor(parentCollection.key, null, upserted, Date.now());
|
|
10886
|
-
if (!sawBillingLimit) for (const parentId of parentIds) {
|
|
10887
|
-
for (const request of collection.requests) {
|
|
10888
|
-
const result = await walkRequest({
|
|
10889
|
-
deps,
|
|
10890
|
-
collection,
|
|
10891
|
-
request,
|
|
10892
|
-
tracker,
|
|
10893
|
-
billingLimitCredits: opts.billingLimitCredits,
|
|
10894
|
-
since: opts.since,
|
|
10895
|
-
useCheckpoint: false,
|
|
10896
|
-
parentId,
|
|
10897
|
-
parentIdToken: collection.parentIdToken,
|
|
10898
|
-
onProgress: opts.onProgress
|
|
10899
|
-
});
|
|
10900
|
-
addResult(result);
|
|
10901
|
-
if (result.stoppedReason === "billing-limit") break;
|
|
10902
|
-
}
|
|
10903
|
-
if (sawBillingLimit) break;
|
|
10904
|
-
}
|
|
10905
|
-
deps.cache.setCursor(collection.key, null, upserted, Date.now());
|
|
10906
|
-
} else {
|
|
10907
|
-
const useCheckpoint = collection.requests.length === 1;
|
|
10908
|
-
for (const request of collection.requests) {
|
|
10909
|
-
const result = await walkRequest({
|
|
10910
|
-
deps,
|
|
10911
|
-
collection,
|
|
10912
|
-
request,
|
|
10913
|
-
tracker,
|
|
10914
|
-
billingLimitCredits: opts.billingLimitCredits,
|
|
10915
|
-
since: opts.since,
|
|
10916
|
-
useCheckpoint,
|
|
10917
|
-
onProgress: opts.onProgress
|
|
10918
|
-
});
|
|
10919
|
-
addResult(result);
|
|
10920
|
-
if (useCheckpoint) persistSingleRequestCheckpoint({
|
|
10921
|
-
cache: deps.cache,
|
|
10922
|
-
collection,
|
|
10923
|
-
result
|
|
10924
|
-
});
|
|
10925
|
-
if (result.stoppedReason === "billing-limit") break;
|
|
10926
|
-
}
|
|
10927
|
-
if (!useCheckpoint) deps.cache.setCursor(collection.key, null, upserted, Date.now());
|
|
10928
|
-
}
|
|
10929
|
-
const creditsSpent = tracker.spent();
|
|
10930
|
-
return {
|
|
10931
|
-
collection: collection.key,
|
|
10932
|
-
objectsUpserted: upserted,
|
|
10933
|
-
pages,
|
|
10934
|
-
creditsSpent,
|
|
10935
|
-
usdSpent: creditsToUSD(creditsSpent),
|
|
10936
|
-
stoppedReason,
|
|
10937
|
-
partial: sawBillingLimit
|
|
10938
|
-
};
|
|
11193
|
+
const parseSince = (input, now = Date.now()) => {
|
|
11194
|
+
const value = input.trim();
|
|
11195
|
+
const duration = durationPattern.exec(value);
|
|
11196
|
+
if (duration != null) return now - Number.parseInt(duration[1] ?? "", 10) * unitMs[duration[2]];
|
|
11197
|
+
const parsed = isoPattern.test(value) ? Date.parse(value) : NaN;
|
|
11198
|
+
if (Number.isNaN(parsed)) throw new Error("Invalid --since value. Use an ISO 8601 date/datetime like 2026-06-01 or 2026-06-01T00:00:00Z, or a compact duration like 2:days, 3:weeks, or 5:hours.");
|
|
11199
|
+
return parsed;
|
|
10939
11200
|
};
|
|
10940
11201
|
//#endregion
|
|
10941
11202
|
//#region ../../node_modules/.bun/citty@0.2.2/node_modules/citty/dist/_chunks/libs/scule.mjs
|
|
@@ -11435,7 +11696,239 @@ const { costBPSForCategory: costBPSForCategory$1 } = createPricing([
|
|
|
11435
11696
|
"unitPriceUSD": .001,
|
|
11436
11697
|
"featureId": "owned_read_resource"
|
|
11437
11698
|
}
|
|
11438
|
-
]);
|
|
11699
|
+
]);
|
|
11700
|
+
//#endregion
|
|
11701
|
+
//#region ../../packages/lib/src/routes.ts
|
|
11702
|
+
const colonParamSegmentPattern = /^:(?<name>[A-Za-z0-9_]+)$/;
|
|
11703
|
+
const braceParamSegmentPattern = /^\{(?<name>[A-Za-z0-9_]+)\}$/;
|
|
11704
|
+
const splitPathSegments = (path) => path.split("/").filter(Boolean);
|
|
11705
|
+
const classifyPathSegment = (segment) => {
|
|
11706
|
+
if (segment === "*") return {
|
|
11707
|
+
kind: "wildcard",
|
|
11708
|
+
segment
|
|
11709
|
+
};
|
|
11710
|
+
const colonParam = segment.match(colonParamSegmentPattern)?.groups?.name;
|
|
11711
|
+
if (colonParam) return {
|
|
11712
|
+
kind: "param",
|
|
11713
|
+
name: colonParam,
|
|
11714
|
+
segment,
|
|
11715
|
+
syntax: "colon"
|
|
11716
|
+
};
|
|
11717
|
+
const braceParam = segment.match(braceParamSegmentPattern)?.groups?.name;
|
|
11718
|
+
if (braceParam) return {
|
|
11719
|
+
kind: "param",
|
|
11720
|
+
name: braceParam,
|
|
11721
|
+
segment,
|
|
11722
|
+
syntax: "brace"
|
|
11723
|
+
};
|
|
11724
|
+
return {
|
|
11725
|
+
kind: "literal",
|
|
11726
|
+
segment
|
|
11727
|
+
};
|
|
11728
|
+
};
|
|
11729
|
+
const pathTemplateValueFrom = (values, name) => {
|
|
11730
|
+
const map = values;
|
|
11731
|
+
return typeof map.get === "function" ? map.get(name) : values[name];
|
|
11732
|
+
};
|
|
11733
|
+
const safePathSegment = (value) => {
|
|
11734
|
+
const segment = String(value);
|
|
11735
|
+
try {
|
|
11736
|
+
return encodeURIComponent(decodeURIComponent(segment));
|
|
11737
|
+
} catch {
|
|
11738
|
+
return encodeURIComponent(segment);
|
|
11739
|
+
}
|
|
11740
|
+
};
|
|
11741
|
+
const defaultMissingParamMessage = (name) => `Missing path parameter: ${name}`;
|
|
11742
|
+
const resolvePathTemplateWith = (template, valueFrom, options = {}) => {
|
|
11743
|
+
const missingParamMessage = options.missingParamMessage ?? defaultMissingParamMessage;
|
|
11744
|
+
const resolvedPath = splitPathSegments(template).map((segment) => {
|
|
11745
|
+
const classified = classifyPathSegment(segment);
|
|
11746
|
+
if (classified.kind !== "param") return segment;
|
|
11747
|
+
const value = valueFrom(classified.name);
|
|
11748
|
+
if (value === void 0) throw new Error(missingParamMessage(classified.name));
|
|
11749
|
+
return safePathSegment(value);
|
|
11750
|
+
}).join("/");
|
|
11751
|
+
if (!resolvedPath) return template.startsWith("/") ? "/" : "";
|
|
11752
|
+
return `${template.startsWith("/") ? "/" : ""}${resolvedPath}`;
|
|
11753
|
+
};
|
|
11754
|
+
const resolvePathTemplate = (template, values, options) => resolvePathTemplateWith(template, (name) => pathTemplateValueFrom(values, name), options);
|
|
11755
|
+
//#endregion
|
|
11756
|
+
//#region ../../packages/api/src/proxy/types.ts
|
|
11757
|
+
const Capability$1 = _enum(["read", "write"]);
|
|
11758
|
+
const EndpointPagination = object({
|
|
11759
|
+
style: _enum([
|
|
11760
|
+
"none",
|
|
11761
|
+
"cursor",
|
|
11762
|
+
"offset"
|
|
11763
|
+
]),
|
|
11764
|
+
limitParam: string().optional(),
|
|
11765
|
+
cursorParam: string().optional(),
|
|
11766
|
+
offsetParam: string().optional(),
|
|
11767
|
+
cursorPath: string().optional(),
|
|
11768
|
+
previousCursorPath: string().optional(),
|
|
11769
|
+
resultCountPath: string().optional(),
|
|
11770
|
+
totalCountPath: string().optional(),
|
|
11771
|
+
itemsPath: string().optional(),
|
|
11772
|
+
pagingPath: string().optional()
|
|
11773
|
+
});
|
|
11774
|
+
object({
|
|
11775
|
+
method: _enum([
|
|
11776
|
+
"GET",
|
|
11777
|
+
"POST",
|
|
11778
|
+
"PATCH",
|
|
11779
|
+
"DELETE",
|
|
11780
|
+
"PUT"
|
|
11781
|
+
]),
|
|
11782
|
+
path: string(),
|
|
11783
|
+
upstreamMethod: _enum([
|
|
11784
|
+
"GET",
|
|
11785
|
+
"POST",
|
|
11786
|
+
"PATCH",
|
|
11787
|
+
"DELETE",
|
|
11788
|
+
"PUT"
|
|
11789
|
+
]).optional(),
|
|
11790
|
+
upstreamPath: string().optional(),
|
|
11791
|
+
capability: Capability$1,
|
|
11792
|
+
mutates: boolean$1().optional(),
|
|
11793
|
+
pagination: EndpointPagination.optional(),
|
|
11794
|
+
costBPS: number().int().nonnegative(),
|
|
11795
|
+
billingCategory: string(),
|
|
11796
|
+
billingOwnedCategory: string().optional(),
|
|
11797
|
+
billingURLCategory: string().optional(),
|
|
11798
|
+
maxUsageUnits: number().int().positive().optional(),
|
|
11799
|
+
kind: string().optional()
|
|
11800
|
+
});
|
|
11801
|
+
//#endregion
|
|
11802
|
+
//#region ../../packages/lib/src/url.ts
|
|
11803
|
+
const stripTrailingSlashes = (value) => value.replace(/\/+$/, "");
|
|
11804
|
+
const stripLeadingSlashes = (value) => value.replace(/^\/+/, "");
|
|
11805
|
+
const ensureTrailingSlash = (value) => value.endsWith("/") ? value : `${value}/`;
|
|
11806
|
+
const absoluteURLPathPattern = /^[a-z][a-z0-9+.-]*:/i;
|
|
11807
|
+
const protocolRelativeURLPathPattern = /^\/\//;
|
|
11808
|
+
const createURL = (baseURL, path) => {
|
|
11809
|
+
const trimmedPath = path.trimStart();
|
|
11810
|
+
if (absoluteURLPathPattern.test(trimmedPath) || protocolRelativeURLPathPattern.test(trimmedPath)) throw new Error("createURL: path must be relative, not absolute or protocol-relative");
|
|
11811
|
+
return new URL(stripLeadingSlashes(path), ensureTrailingSlash(stripTrailingSlashes(baseURL.toString())));
|
|
11812
|
+
};
|
|
11813
|
+
const createAbsoluteURL = (baseURL, path) => createURL(baseURL, path).toString();
|
|
11814
|
+
//#endregion
|
|
11815
|
+
//#region ../../packages/linkedin/src/profiles.ts
|
|
11816
|
+
const displayNameFrom = (firstName, lastName, handle) => [firstName, lastName].filter(Boolean).join(" ").trim() || handle || "";
|
|
11817
|
+
const rowFrom$1 = (fields) => {
|
|
11818
|
+
const providerId = fields.providerId?.trim();
|
|
11819
|
+
if (!providerId) return null;
|
|
11820
|
+
const handle = fields.handle?.trim() || null;
|
|
11821
|
+
const displayName = fields.displayName?.trim() || displayNameFrom(fields.firstName, fields.lastName, handle ?? void 0);
|
|
11822
|
+
if (!displayName) return null;
|
|
11823
|
+
return {
|
|
11824
|
+
providerId,
|
|
11825
|
+
handle,
|
|
11826
|
+
displayName,
|
|
11827
|
+
avatarURL: fields.avatarURL?.trim() || null,
|
|
11828
|
+
headline: fields.headline?.trim() || null,
|
|
11829
|
+
raw: fields.raw
|
|
11830
|
+
};
|
|
11831
|
+
};
|
|
11832
|
+
const trimmed = (value) => value?.trim() || void 0;
|
|
11833
|
+
const LinkedinUserSpecifics = looseObject({
|
|
11834
|
+
member_id: string().optional(),
|
|
11835
|
+
description: string().nullish(),
|
|
11836
|
+
headline: string().nullish(),
|
|
11837
|
+
occupation: string().nullish()
|
|
11838
|
+
});
|
|
11839
|
+
const LinkedinUser = looseObject({
|
|
11840
|
+
id: string().optional(),
|
|
11841
|
+
public_identifier: string().optional(),
|
|
11842
|
+
display_name: string().optional(),
|
|
11843
|
+
first_name: string().optional(),
|
|
11844
|
+
last_name: string().optional(),
|
|
11845
|
+
public_picture_url: string().nullish(),
|
|
11846
|
+
description: string().nullish(),
|
|
11847
|
+
headline: string().nullish(),
|
|
11848
|
+
specifics: LinkedinUserSpecifics.nullish()
|
|
11849
|
+
});
|
|
11850
|
+
const providerIdFromUser = (user) => trimmed(user.id) ?? trimmed(user.specifics?.member_id);
|
|
11851
|
+
const headlineFromUser = (user) => trimmed(user.description) ?? trimmed(user.headline) ?? trimmed(user.specifics?.description) ?? trimmed(user.specifics?.headline) ?? trimmed(user.specifics?.occupation) ?? null;
|
|
11852
|
+
const rowFromUser = ({ user, providerId, raw }) => rowFrom$1({
|
|
11853
|
+
providerId: providerId ?? providerIdFromUser(user),
|
|
11854
|
+
displayName: user.display_name,
|
|
11855
|
+
firstName: user.first_name,
|
|
11856
|
+
lastName: user.last_name,
|
|
11857
|
+
handle: user.public_identifier,
|
|
11858
|
+
avatarURL: user.public_picture_url,
|
|
11859
|
+
headline: headlineFromUser(user),
|
|
11860
|
+
raw
|
|
11861
|
+
});
|
|
11862
|
+
const ProfileBody = LinkedinUser;
|
|
11863
|
+
const extractProfile$1 = (body) => {
|
|
11864
|
+
const parsed = ProfileBody.safeParse(body);
|
|
11865
|
+
if (!parsed.success) return [];
|
|
11866
|
+
const row = rowFrom$1({
|
|
11867
|
+
providerId: providerIdFromUser(parsed.data),
|
|
11868
|
+
displayName: parsed.data.display_name,
|
|
11869
|
+
firstName: parsed.data.first_name,
|
|
11870
|
+
lastName: parsed.data.last_name,
|
|
11871
|
+
handle: parsed.data.public_identifier,
|
|
11872
|
+
avatarURL: parsed.data.public_picture_url,
|
|
11873
|
+
headline: headlineFromUser(parsed.data),
|
|
11874
|
+
raw: parsed.data
|
|
11875
|
+
});
|
|
11876
|
+
return row ? [row] : [];
|
|
11877
|
+
};
|
|
11878
|
+
const RelationsBody = looseObject({ data: array(looseObject({
|
|
11879
|
+
id: string().optional(),
|
|
11880
|
+
object: string().optional(),
|
|
11881
|
+
specifics: LinkedinUserSpecifics.nullish(),
|
|
11882
|
+
user: LinkedinUser.nullish()
|
|
11883
|
+
})) });
|
|
11884
|
+
const extractConnections = (body) => {
|
|
11885
|
+
const parsed = RelationsBody.safeParse(body);
|
|
11886
|
+
if (!parsed.success) return [];
|
|
11887
|
+
return parsed.data.data.flatMap((item) => {
|
|
11888
|
+
const user = item.user ?? (item.object === "UserRelation" ? null : item);
|
|
11889
|
+
if (!user) {
|
|
11890
|
+
const row = rowFrom$1({
|
|
11891
|
+
providerId: trimmed(item.specifics?.member_id),
|
|
11892
|
+
raw: item
|
|
11893
|
+
});
|
|
11894
|
+
return row ? [row] : [];
|
|
11895
|
+
}
|
|
11896
|
+
const row = rowFromUser({
|
|
11897
|
+
user,
|
|
11898
|
+
providerId: providerIdFromUser(user) ?? trimmed(item.specifics?.member_id) ?? (item.object === "UserRelation" ? void 0 : trimmed(item.id)),
|
|
11899
|
+
raw: item
|
|
11900
|
+
});
|
|
11901
|
+
return row ? [row] : [];
|
|
11902
|
+
});
|
|
11903
|
+
};
|
|
11904
|
+
const GroupParticipant = looseObject({
|
|
11905
|
+
object: string().optional(),
|
|
11906
|
+
is_self: boolean$1().optional(),
|
|
11907
|
+
is_admin: boolean$1().optional(),
|
|
11908
|
+
user: LinkedinUser
|
|
11909
|
+
});
|
|
11910
|
+
looseObject({ data: array(looseObject({
|
|
11911
|
+
id: string().optional(),
|
|
11912
|
+
name: string().optional(),
|
|
11913
|
+
headline: string().nullish(),
|
|
11914
|
+
image_url: string().nullish(),
|
|
11915
|
+
user_id: string().nullish(),
|
|
11916
|
+
is_group: boolean$1().optional(),
|
|
11917
|
+
is_1to1: boolean$1().optional(),
|
|
11918
|
+
type: string().optional(),
|
|
11919
|
+
participants_count: number().optional(),
|
|
11920
|
+
user: LinkedinUser.nullish(),
|
|
11921
|
+
participants: array(GroupParticipant).nullish()
|
|
11922
|
+
})) });
|
|
11923
|
+
looseObject({
|
|
11924
|
+
data: array(GroupParticipant),
|
|
11925
|
+
next_cursor: string().nullish()
|
|
11926
|
+
});
|
|
11927
|
+
const linkedinProfileExtractors = {
|
|
11928
|
+
"GET /users/me": extractProfile$1,
|
|
11929
|
+
"GET /users/:identifier": extractProfile$1,
|
|
11930
|
+
"GET /users/connections": extractConnections
|
|
11931
|
+
};
|
|
11439
11932
|
const LINKEDIN_OPERATIONS = [
|
|
11440
11933
|
{
|
|
11441
11934
|
schemaBaseName: "MessagesList",
|
|
@@ -12130,8 +12623,8 @@ const LINKEDIN_OPERATIONS = [
|
|
|
12130
12623
|
totalCountPath: "total_count",
|
|
12131
12624
|
itemsPath: "data"
|
|
12132
12625
|
},
|
|
12133
|
-
costBPS:
|
|
12134
|
-
billingCategory: "
|
|
12626
|
+
costBPS: 10,
|
|
12627
|
+
billingCategory: "owned_read",
|
|
12135
12628
|
kind: "users"
|
|
12136
12629
|
},
|
|
12137
12630
|
parameters: [
|
|
@@ -12533,61 +13026,6 @@ function kebabCase(str) {
|
|
|
12533
13026
|
//#region ../../packages/codegen/src/naming.ts
|
|
12534
13027
|
const flagName = (name) => kebabCase(name.replace(/[._]/g, "-"));
|
|
12535
13028
|
//#endregion
|
|
12536
|
-
//#region ../../packages/lib/src/routes.ts
|
|
12537
|
-
const colonParamSegmentPattern = /^:(?<name>[A-Za-z0-9_]+)$/;
|
|
12538
|
-
const braceParamSegmentPattern = /^\{(?<name>[A-Za-z0-9_]+)\}$/;
|
|
12539
|
-
const splitPathSegments = (path) => path.split("/").filter(Boolean);
|
|
12540
|
-
const classifyPathSegment = (segment) => {
|
|
12541
|
-
if (segment === "*") return {
|
|
12542
|
-
kind: "wildcard",
|
|
12543
|
-
segment
|
|
12544
|
-
};
|
|
12545
|
-
const colonParam = segment.match(colonParamSegmentPattern)?.groups?.name;
|
|
12546
|
-
if (colonParam) return {
|
|
12547
|
-
kind: "param",
|
|
12548
|
-
name: colonParam,
|
|
12549
|
-
segment,
|
|
12550
|
-
syntax: "colon"
|
|
12551
|
-
};
|
|
12552
|
-
const braceParam = segment.match(braceParamSegmentPattern)?.groups?.name;
|
|
12553
|
-
if (braceParam) return {
|
|
12554
|
-
kind: "param",
|
|
12555
|
-
name: braceParam,
|
|
12556
|
-
segment,
|
|
12557
|
-
syntax: "brace"
|
|
12558
|
-
};
|
|
12559
|
-
return {
|
|
12560
|
-
kind: "literal",
|
|
12561
|
-
segment
|
|
12562
|
-
};
|
|
12563
|
-
};
|
|
12564
|
-
const pathTemplateValueFrom = (values, name) => {
|
|
12565
|
-
const map = values;
|
|
12566
|
-
return typeof map.get === "function" ? map.get(name) : values[name];
|
|
12567
|
-
};
|
|
12568
|
-
const safePathSegment = (value) => {
|
|
12569
|
-
const segment = String(value);
|
|
12570
|
-
try {
|
|
12571
|
-
return encodeURIComponent(decodeURIComponent(segment));
|
|
12572
|
-
} catch {
|
|
12573
|
-
return encodeURIComponent(segment);
|
|
12574
|
-
}
|
|
12575
|
-
};
|
|
12576
|
-
const defaultMissingParamMessage = (name) => `Missing path parameter: ${name}`;
|
|
12577
|
-
const resolvePathTemplateWith = (template, valueFrom, options = {}) => {
|
|
12578
|
-
const missingParamMessage = options.missingParamMessage ?? defaultMissingParamMessage;
|
|
12579
|
-
const resolvedPath = splitPathSegments(template).map((segment) => {
|
|
12580
|
-
const classified = classifyPathSegment(segment);
|
|
12581
|
-
if (classified.kind !== "param") return segment;
|
|
12582
|
-
const value = valueFrom(classified.name);
|
|
12583
|
-
if (value === void 0) throw new Error(missingParamMessage(classified.name));
|
|
12584
|
-
return safePathSegment(value);
|
|
12585
|
-
}).join("/");
|
|
12586
|
-
if (!resolvedPath) return template.startsWith("/") ? "/" : "";
|
|
12587
|
-
return `${template.startsWith("/") ? "/" : ""}${resolvedPath}`;
|
|
12588
|
-
};
|
|
12589
|
-
const resolvePathTemplate = (template, values, options) => resolvePathTemplateWith(template, (name) => pathTemplateValueFrom(values, name), options);
|
|
12590
|
-
//#endregion
|
|
12591
13029
|
//#region ../../packages/linkedin/src/cache.ts
|
|
12592
13030
|
const linkedinCacheableEndpointPaths = [
|
|
12593
13031
|
"GET /companies/:identifier",
|
|
@@ -13575,28 +14013,102 @@ const commandOrder = {
|
|
|
13575
14013
|
};
|
|
13576
14014
|
const hiddenGroups = new Set(["search"]);
|
|
13577
14015
|
const pascalCase = (value) => value.split(/[^A-Za-z0-9]+/).filter(Boolean).map((part) => part.charAt(0).toUpperCase() + part.slice(1)).join("");
|
|
13578
|
-
const confirmTTY$1 = async (message) => {
|
|
13579
|
-
const readline = createInterface({
|
|
13580
|
-
input: process.stdin,
|
|
13581
|
-
output: process.stdout
|
|
13582
|
-
});
|
|
13583
|
-
try {
|
|
13584
|
-
const answer = await readline.question(`${message} [y/N] `);
|
|
13585
|
-
return /^(y|yes)$/i.test(answer.trim());
|
|
13586
|
-
} finally {
|
|
13587
|
-
readline.close();
|
|
13588
|
-
}
|
|
13589
|
-
};
|
|
13590
14016
|
const requireAccountSelector$1 = (selector) => {
|
|
13591
14017
|
if (selector === void 0) throw new UsageError("Selected LinkedIn account is missing an account selector.");
|
|
13592
14018
|
return selector;
|
|
13593
14019
|
};
|
|
14020
|
+
const numberFromPath = (value, path) => {
|
|
14021
|
+
let current = value;
|
|
14022
|
+
for (const segment of path.split(".")) {
|
|
14023
|
+
if (!isRecord$5(current)) return null;
|
|
14024
|
+
current = current[segment];
|
|
14025
|
+
}
|
|
14026
|
+
return typeof current === "number" && Number.isFinite(current) ? current : null;
|
|
14027
|
+
};
|
|
14028
|
+
const linkedInCountFields = { li_connections: [
|
|
14029
|
+
"specifics.relations_count",
|
|
14030
|
+
"specifics.connections_count",
|
|
14031
|
+
"relations_count",
|
|
14032
|
+
"connections_count",
|
|
14033
|
+
"connection_count"
|
|
14034
|
+
] };
|
|
14035
|
+
const ownProfileCount = async (args) => {
|
|
14036
|
+
const fields = linkedInCountFields[args.collectionKey];
|
|
14037
|
+
if (fields === void 0) return null;
|
|
14038
|
+
const body = await (await args.deps.api.get(`linkedin/users/${encodeURIComponent(args.ownId)}`, { headers: args.accountHeaders })).json();
|
|
14039
|
+
for (const field of fields) {
|
|
14040
|
+
const count = numberFromPath(body, field);
|
|
14041
|
+
if (count != null) return count;
|
|
14042
|
+
}
|
|
14043
|
+
return null;
|
|
14044
|
+
};
|
|
14045
|
+
const syncQuoteLine$1 = (estCredits, quoteLine) => `Estimated ~${Math.ceil(estCredits)} credits.\n${quoteLine}\n`;
|
|
14046
|
+
const autoCapPartialLine$1 = () => `Hit the ${AUTO_CAP_CREDITS} credit auto-cap (partial). Re-run with --credits <N> to continue.\n`;
|
|
14047
|
+
const syncProgressMessage = (progress) => `${progress.collectionKey}: ${progress.pagesCompleted} page(s), ${progress.objectsUpserted} object(s), ${progress.creditsSpent} credits.`;
|
|
14048
|
+
const syncCompleteMessage = (summary) => `Synced ${summary.collection}: ${summary.pages} page(s), ${summary.objectsUpserted} object(s), ${summary.creditsSpent} credits.`;
|
|
14049
|
+
const createSyncProgressReporter = (deps, collectionKey) => {
|
|
14050
|
+
if ((deps.isInteractiveTerminal?.() ?? process.stdout.isTTY === true) && deps.ui) {
|
|
14051
|
+
const spinner = deps.ui.spinner();
|
|
14052
|
+
spinner.start(`Syncing ${collectionKey}...`);
|
|
14053
|
+
return {
|
|
14054
|
+
error: () => spinner.error(`Sync failed for ${collectionKey}.`),
|
|
14055
|
+
onPage: (progress) => spinner.message(syncProgressMessage(progress)),
|
|
14056
|
+
stop: (summary) => spinner.stop(syncCompleteMessage(summary))
|
|
14057
|
+
};
|
|
14058
|
+
}
|
|
14059
|
+
return {
|
|
14060
|
+
error: () => void 0,
|
|
14061
|
+
onPage: (progress) => process.stderr.write(`${syncProgressMessage(progress)}\n`),
|
|
14062
|
+
stop: () => void 0
|
|
14063
|
+
};
|
|
14064
|
+
};
|
|
13594
14065
|
const buildSync$1 = (deps, platform) => defineCommand({
|
|
13595
|
-
meta: {
|
|
14066
|
+
meta: commandMeta$2({
|
|
13596
14067
|
name: "sync",
|
|
13597
14068
|
description: "Sync a collection into the local cache (spends credits)",
|
|
13598
|
-
|
|
13599
|
-
|
|
14069
|
+
method: "GET",
|
|
14070
|
+
capability: "read",
|
|
14071
|
+
mutates: true,
|
|
14072
|
+
schemaContract: {
|
|
14073
|
+
auth: {
|
|
14074
|
+
required: true,
|
|
14075
|
+
provider: "linkedin",
|
|
14076
|
+
scopes: ["read"]
|
|
14077
|
+
},
|
|
14078
|
+
capability: "read",
|
|
14079
|
+
confirmation: {
|
|
14080
|
+
required: true,
|
|
14081
|
+
reason: "Sync can spend usage credits and writes local cache state."
|
|
14082
|
+
},
|
|
14083
|
+
idempotency: {
|
|
14084
|
+
safe: false,
|
|
14085
|
+
retry: "never",
|
|
14086
|
+
reason: "Repeated syncs can spend additional usage credits and update local rows."
|
|
14087
|
+
},
|
|
14088
|
+
method: "GET",
|
|
14089
|
+
mutates: true,
|
|
14090
|
+
outputSchema: unknown(),
|
|
14091
|
+
pagination: { style: "none" },
|
|
14092
|
+
responseShaping: {
|
|
14093
|
+
supported: true,
|
|
14094
|
+
formats: ["json"],
|
|
14095
|
+
projection: true
|
|
14096
|
+
},
|
|
14097
|
+
examples: [{ command: "social linkedin sync connections --credits 1000" }],
|
|
14098
|
+
cost: {
|
|
14099
|
+
metered: true,
|
|
14100
|
+
unit: "usage_credit",
|
|
14101
|
+
notes: ["Upstream reads during sync are metered."]
|
|
14102
|
+
},
|
|
14103
|
+
hazards: [{
|
|
14104
|
+
code: "local_cache_write",
|
|
14105
|
+
message: "Writes synced LinkedIn rows and cursor state to the local cache."
|
|
14106
|
+
}, {
|
|
14107
|
+
code: "usage_credit_spend",
|
|
14108
|
+
message: "May spend usage credits while reading upstream LinkedIn data."
|
|
14109
|
+
}]
|
|
14110
|
+
}
|
|
14111
|
+
}),
|
|
13600
14112
|
args: {
|
|
13601
14113
|
collection: {
|
|
13602
14114
|
type: "positional",
|
|
@@ -13607,15 +14119,13 @@ const buildSync$1 = (deps, platform) => defineCommand({
|
|
|
13607
14119
|
type: "string",
|
|
13608
14120
|
description: "ISO date/datetime or compact duration like 2:days|3:weeks|5:hours"
|
|
13609
14121
|
},
|
|
13610
|
-
|
|
14122
|
+
credits: {
|
|
13611
14123
|
type: "string",
|
|
13612
|
-
|
|
13613
|
-
description: "Hard cap on spend in USD (default 10)"
|
|
14124
|
+
description: "Consent and hard cap in credits"
|
|
13614
14125
|
},
|
|
13615
|
-
|
|
14126
|
+
reset: {
|
|
13616
14127
|
type: "boolean",
|
|
13617
|
-
|
|
13618
|
-
description: "Skip the TTY confirmation"
|
|
14128
|
+
description: "Delete local rows and sync state for the selected collection"
|
|
13619
14129
|
},
|
|
13620
14130
|
account: {
|
|
13621
14131
|
type: "string",
|
|
@@ -13623,7 +14133,7 @@ const buildSync$1 = (deps, platform) => defineCommand({
|
|
|
13623
14133
|
}
|
|
13624
14134
|
},
|
|
13625
14135
|
run: async ({ args }) => {
|
|
13626
|
-
const collections =
|
|
14136
|
+
const collections = syncableCollectionsFor(platform);
|
|
13627
14137
|
const shortName = (collection) => collection.key.replace(/^(x|li)_/, "");
|
|
13628
14138
|
const account = typeof args.account === "string" ? args.account : void 0;
|
|
13629
14139
|
const res = await deps.resolveOwnLinkedinAccount({ account });
|
|
@@ -13631,6 +14141,7 @@ const buildSync$1 = (deps, platform) => defineCommand({
|
|
|
13631
14141
|
const accountSelector = requireAccountSelector$1(res.account.selector);
|
|
13632
14142
|
const accountHeaders = { "x-social-account-id": accountSelector };
|
|
13633
14143
|
if (!args.collection) {
|
|
14144
|
+
if (args.reset === true) throw new UsageError("--reset requires a collection");
|
|
13634
14145
|
const cache = openCache({
|
|
13635
14146
|
platform,
|
|
13636
14147
|
accountId: accountSelector
|
|
@@ -13651,43 +14162,102 @@ const buildSync$1 = (deps, platform) => defineCommand({
|
|
|
13651
14162
|
}
|
|
13652
14163
|
const collection = collections.find((candidate) => shortName(candidate) === args.collection || candidate.key === args.collection);
|
|
13653
14164
|
if (!collection) throw new UsageError(`Unknown collection "${args.collection}". Available: ${collections.map(shortName).join(", ")}`);
|
|
14165
|
+
if (args.reset === true) {
|
|
14166
|
+
if (args.credits != null) throw new UsageError("--reset cannot be combined with --credits");
|
|
14167
|
+
if (args.since != null) throw new UsageError("--reset cannot be combined with --since");
|
|
14168
|
+
const cache = openCache({
|
|
14169
|
+
platform,
|
|
14170
|
+
accountId: accountSelector
|
|
14171
|
+
});
|
|
14172
|
+
const resetCollections = resetTablesFor(collection);
|
|
14173
|
+
try {
|
|
14174
|
+
deps.printData({
|
|
14175
|
+
collection: collection.key,
|
|
14176
|
+
reset: true,
|
|
14177
|
+
tables: cache.reset({
|
|
14178
|
+
tables: resetCollections.map((resetCollection) => resetCollection.key),
|
|
14179
|
+
collections: resetCollections.map((resetCollection) => resetCollection.key)
|
|
14180
|
+
})
|
|
14181
|
+
});
|
|
14182
|
+
} finally {
|
|
14183
|
+
cache.close();
|
|
14184
|
+
}
|
|
14185
|
+
return;
|
|
14186
|
+
}
|
|
13654
14187
|
let since;
|
|
13655
14188
|
if (args.since != null) {
|
|
13656
14189
|
if (!collection.sinceField) throw new UsageError(`${shortName(collection)} does not support --since`);
|
|
14190
|
+
if (collection.requiresFullInitialSync) {
|
|
14191
|
+
const cache = openCache({
|
|
14192
|
+
platform,
|
|
14193
|
+
accountId: accountSelector
|
|
14194
|
+
});
|
|
14195
|
+
try {
|
|
14196
|
+
if (cache.getCursor(collection.key)?.lastSynced == null) throw new UsageError(`${shortName(collection)} requires a full initial sync before --since`);
|
|
14197
|
+
} finally {
|
|
14198
|
+
cache.close();
|
|
14199
|
+
}
|
|
14200
|
+
}
|
|
13657
14201
|
since = parseSince(String(args.since));
|
|
13658
14202
|
}
|
|
13659
|
-
const
|
|
13660
|
-
if (!Number.isFinite(
|
|
13661
|
-
const billingLimitCredits = usdToCredits(limitUSD);
|
|
14203
|
+
const creditsArg = args.credits != null && args.credits !== "" ? Number(args.credits) : null;
|
|
14204
|
+
if (creditsArg != null && (!Number.isFinite(creditsArg) || creditsArg < 0)) throw new UsageError("--credits must be a non-negative number of credits");
|
|
13662
14205
|
const cache = openCache({
|
|
13663
14206
|
platform,
|
|
13664
14207
|
accountId: accountSelector
|
|
13665
14208
|
});
|
|
13666
|
-
const
|
|
13667
|
-
|
|
13668
|
-
|
|
13669
|
-
|
|
13670
|
-
|
|
13671
|
-
|
|
14209
|
+
const firstSync = cache.getCursor(collection.key)?.lastSynced == null;
|
|
14210
|
+
const count = firstSync ? await ownProfileCount({
|
|
14211
|
+
deps,
|
|
14212
|
+
collectionKey: collection.key,
|
|
14213
|
+
ownId,
|
|
14214
|
+
accountHeaders
|
|
14215
|
+
}) : null;
|
|
14216
|
+
const plan = planSync({
|
|
14217
|
+
collection,
|
|
14218
|
+
creditsArg,
|
|
14219
|
+
count,
|
|
14220
|
+
firstSync
|
|
14221
|
+
});
|
|
14222
|
+
if (plan.mode === "quote") {
|
|
14223
|
+
cache.close();
|
|
14224
|
+
process.stderr.write(syncQuoteLine$1(plan.estCredits, plan.quoteLine));
|
|
14225
|
+
return;
|
|
14226
|
+
}
|
|
14227
|
+
if (plan.mode === "reject") {
|
|
14228
|
+
cache.close();
|
|
14229
|
+
throw new UsageError(`${shortName(collection)}: ${plan.message}`);
|
|
14230
|
+
}
|
|
14231
|
+
const onEstimate = (estimate) => {
|
|
14232
|
+
const line = estimate.unknown ? `Estimated ~${estimate.perPageCredits} credits/page; total unknown, bounded by ${estimate.limitCredits} credits.` : `Estimated ~${estimate.estCredits} credits across ${estimate.estPages} page(s); limit ${estimate.limitCredits} credits.`;
|
|
13672
14233
|
process.stderr.write(`${line}\n`);
|
|
13673
14234
|
return true;
|
|
13674
14235
|
};
|
|
13675
14236
|
const onProgress = (message) => process.stderr.write(`${message}\n`);
|
|
14237
|
+
const progress = createSyncProgressReporter(deps, collection.key);
|
|
13676
14238
|
try {
|
|
13677
14239
|
const summary = await runSync({
|
|
13678
14240
|
api: deps.api,
|
|
13679
14241
|
cache,
|
|
13680
14242
|
platform,
|
|
13681
14243
|
ownId,
|
|
13682
|
-
accountHeaders
|
|
14244
|
+
accountHeaders,
|
|
14245
|
+
profileExtractors: linkedinProfileExtractors
|
|
13683
14246
|
}, {
|
|
13684
14247
|
collection,
|
|
13685
14248
|
since,
|
|
13686
|
-
|
|
14249
|
+
expectedTotalCount: firstSync ? count : null,
|
|
14250
|
+
billingLimitCredits: plan.billingLimitCredits,
|
|
13687
14251
|
onEstimate,
|
|
14252
|
+
onPage: progress.onPage,
|
|
13688
14253
|
onProgress
|
|
13689
14254
|
});
|
|
14255
|
+
progress.stop(summary);
|
|
14256
|
+
if (plan.capAndGo && summary.stoppedReason === "credit-limit") process.stderr.write(autoCapPartialLine$1());
|
|
13690
14257
|
deps.printData(summary);
|
|
14258
|
+
} catch (error) {
|
|
14259
|
+
progress.error();
|
|
14260
|
+
throw error;
|
|
13691
14261
|
} finally {
|
|
13692
14262
|
cache.close();
|
|
13693
14263
|
}
|
|
@@ -14132,51 +14702,103 @@ const buildMessage$1 = (deps, sendCommand) => {
|
|
|
14132
14702
|
}
|
|
14133
14703
|
});
|
|
14134
14704
|
};
|
|
14135
|
-
const pickedArgs = (args, names) => {
|
|
14136
|
-
const picked = { _: [] };
|
|
14137
|
-
for (const name of names) if (args[name] !== void 0) picked[name] = args[name];
|
|
14138
|
-
return picked;
|
|
14139
|
-
};
|
|
14140
14705
|
const rejectMessagesArgs = (commandName, args, allowed) => rejectUnsupportedArgs(commandName, args, ["target", ...allowed]);
|
|
14141
|
-
const
|
|
14142
|
-
|
|
14143
|
-
|
|
14706
|
+
const cacheReadArgNames = ["account", "limit"];
|
|
14707
|
+
const mustCollection = (key) => {
|
|
14708
|
+
const collection = collectionByKey(key);
|
|
14709
|
+
if (collection === void 0) throw new Error(`Missing sync collection: ${key}`);
|
|
14710
|
+
return collection;
|
|
14711
|
+
};
|
|
14712
|
+
const sqlString = (value) => `'${value.replaceAll("'", "''")}'`;
|
|
14713
|
+
const limitClause = (deps, value) => value === void 0 ? "" : ` LIMIT ${deps.parseIntegerString("--limit", value, { min: 1 })}`;
|
|
14714
|
+
const cacheReadSchemaContract = (command) => {
|
|
14715
|
+
const { inputSchema: _inputSchema, ...schemaContract } = command.meta.schemaContract;
|
|
14716
|
+
const cost = isRecord$5(schemaContract.cost) ? {
|
|
14717
|
+
...schemaContract.cost,
|
|
14718
|
+
cacheable: true,
|
|
14719
|
+
cacheHitCost: 0
|
|
14720
|
+
} : schemaContract.cost;
|
|
14721
|
+
return {
|
|
14722
|
+
...schemaContract,
|
|
14723
|
+
pagination: { style: "none" },
|
|
14724
|
+
cost
|
|
14725
|
+
};
|
|
14726
|
+
};
|
|
14727
|
+
const chatIdFromCachedMessageTarget = (deps, target) => {
|
|
14728
|
+
if (target.startsWith("chat_id:")) return deps.parseResourceId("target", target.slice(8));
|
|
14729
|
+
try {
|
|
14730
|
+
const [, section, subsection, sectionId] = new URL(target).pathname.split("/");
|
|
14731
|
+
if (section === "messaging" && subsection === "thread" && sectionId) return deps.parseResourceId("target", decodeURIComponent(sectionId));
|
|
14732
|
+
} catch {}
|
|
14733
|
+
throw new UsageError("Cached message reads accept chat_id:<id> or a LinkedIn messaging thread URL.");
|
|
14144
14734
|
};
|
|
14145
|
-
const
|
|
14146
|
-
const
|
|
14735
|
+
const withFreshLinkedinCache = async (deps, args, collectionKey, syncName, read, refreshEveryCall = false) => {
|
|
14736
|
+
const account = typeof args.account === "string" ? args.account : void 0;
|
|
14737
|
+
const res = await deps.resolveOwnLinkedinAccount({ account });
|
|
14738
|
+
const ownId = res.providerId;
|
|
14739
|
+
const accountSelector = requireAccountSelector$1(res.account.selector);
|
|
14740
|
+
const accountHeaders = { "x-social-account-id": accountSelector };
|
|
14741
|
+
const cache = openCache({
|
|
14147
14742
|
platform: "linkedin",
|
|
14148
|
-
|
|
14743
|
+
accountId: accountSelector
|
|
14149
14744
|
});
|
|
14150
|
-
|
|
14151
|
-
|
|
14152
|
-
|
|
14153
|
-
|
|
14154
|
-
|
|
14155
|
-
|
|
14156
|
-
|
|
14157
|
-
|
|
14158
|
-
|
|
14159
|
-
|
|
14160
|
-
|
|
14745
|
+
try {
|
|
14746
|
+
const collection = mustCollection(collectionKey);
|
|
14747
|
+
if ((await ensureFreshCache({
|
|
14748
|
+
api: deps.api,
|
|
14749
|
+
cache,
|
|
14750
|
+
platform: "linkedin",
|
|
14751
|
+
ownId,
|
|
14752
|
+
accountHeaders,
|
|
14753
|
+
profileExtractors: linkedinProfileExtractors
|
|
14754
|
+
}, {
|
|
14755
|
+
collection,
|
|
14756
|
+
ttlSeconds: refreshEveryCall ? 0 : void 0
|
|
14757
|
+
})).status === "never-synced") throw new UsageError(`No synced ${syncName} yet. Run \`linkedin sync ${syncName}\` first.`);
|
|
14758
|
+
read(cache);
|
|
14759
|
+
} finally {
|
|
14760
|
+
cache.close();
|
|
14761
|
+
}
|
|
14161
14762
|
};
|
|
14162
|
-
const
|
|
14763
|
+
const readConnections = (deps, args) => withFreshLinkedinCache(deps, args, "li_connections", "connections", (cache) => {
|
|
14764
|
+
deps.printData(cache.query(`SELECT c.*, p.handle AS user_handle, p.display_name AS user_name, p.avatar_url AS user_avatar_url, p.headline AS user_headline FROM li_connections c LEFT JOIN li_profiles p ON c.user_id = p.provider_id ORDER BY c.synced_at DESC${limitClause(deps, args.limit)}`));
|
|
14765
|
+
});
|
|
14766
|
+
const readRequests = (deps, args, requestType) => withFreshLinkedinCache(deps, args, "li_requests", "requests", (cache) => {
|
|
14767
|
+
deps.printData(cache.query(`SELECT r.*, p.handle AS user_handle, p.display_name AS user_name, p.avatar_url AS user_avatar_url, p.headline AS user_headline FROM li_requests r LEFT JOIN li_profiles p ON r.user_id = p.provider_id WHERE r.type = ${sqlString(requestType)} ORDER BY r.synced_at DESC${limitClause(deps, args.limit)}`));
|
|
14768
|
+
}, true);
|
|
14769
|
+
const readPosts = (deps, args) => withFreshLinkedinCache(deps, args, "li_posts", "posts", (cache) => {
|
|
14770
|
+
deps.printData(cache.query(`SELECT * FROM li_posts ORDER BY created_at DESC${limitClause(deps, args.limit)}`));
|
|
14771
|
+
}, true);
|
|
14772
|
+
const buildPosts = (deps, liveCommand) => defineCommand({
|
|
14773
|
+
meta: commandMeta$2({
|
|
14774
|
+
name: "posts",
|
|
14775
|
+
description: "List cached own LinkedIn posts or live target posts [read] | usage: `social linkedin posts [target] --limit N`",
|
|
14776
|
+
method: "GET",
|
|
14777
|
+
capability: "read",
|
|
14778
|
+
mutates: false,
|
|
14779
|
+
schemaContract: liveCommand.meta.schemaContract
|
|
14780
|
+
}),
|
|
14781
|
+
args: liveCommand.args,
|
|
14782
|
+
run: async ({ args, rawArgs }) => {
|
|
14783
|
+
const parsedArgs = args;
|
|
14784
|
+
const target = typeof args.target === "string" ? args.target : Array.isArray(args._) && typeof args._[0] === "string" ? args._[0] : void 0;
|
|
14785
|
+
if (target === void 0) {
|
|
14786
|
+
rejectUnsupportedArgs("linkedin posts", parsedArgs, cacheReadArgNames);
|
|
14787
|
+
await readPosts(deps, parsedArgs);
|
|
14788
|
+
return;
|
|
14789
|
+
}
|
|
14790
|
+
await liveCommand.run?.({
|
|
14791
|
+
rawArgs,
|
|
14792
|
+
args: {
|
|
14793
|
+
...parsedArgs,
|
|
14794
|
+
target
|
|
14795
|
+
},
|
|
14796
|
+
cmd: liveCommand
|
|
14797
|
+
});
|
|
14798
|
+
}
|
|
14799
|
+
});
|
|
14800
|
+
const buildMessages$1 = (deps, listCommand) => {
|
|
14163
14801
|
const markCommand = buildMark(deps);
|
|
14164
|
-
const listArgNames = [
|
|
14165
|
-
"account",
|
|
14166
|
-
"cursor",
|
|
14167
|
-
"before",
|
|
14168
|
-
"after",
|
|
14169
|
-
"limit",
|
|
14170
|
-
"header"
|
|
14171
|
-
];
|
|
14172
|
-
const readArgNames = [
|
|
14173
|
-
"account",
|
|
14174
|
-
"cursor",
|
|
14175
|
-
"before",
|
|
14176
|
-
"after",
|
|
14177
|
-
"limit",
|
|
14178
|
-
"header"
|
|
14179
|
-
];
|
|
14180
14802
|
const markArgNames = ["account"];
|
|
14181
14803
|
return defineCommand({
|
|
14182
14804
|
meta: commandMeta$2({
|
|
@@ -14185,43 +14807,26 @@ const buildMessages$1 = (deps, listCommand, readCommand) => {
|
|
|
14185
14807
|
method: "GET",
|
|
14186
14808
|
capability: "read",
|
|
14187
14809
|
mutates: false,
|
|
14188
|
-
schemaContract: listCommand
|
|
14810
|
+
schemaContract: cacheReadSchemaContract(listCommand),
|
|
14189
14811
|
schemaSubCommands: { mark: markCommand }
|
|
14190
14812
|
}),
|
|
14191
14813
|
args: {
|
|
14192
14814
|
target: {
|
|
14193
14815
|
type: "positional",
|
|
14194
14816
|
required: false,
|
|
14195
|
-
description: "LinkedIn conversation
|
|
14817
|
+
description: "LinkedIn conversation target: chat_id:<id> or a messaging thread URL",
|
|
14196
14818
|
valueHint: "target"
|
|
14197
14819
|
},
|
|
14198
14820
|
account: {
|
|
14199
14821
|
type: "string",
|
|
14200
14822
|
description: "Account selector: @handle or profile_id:<id>; overrides config default"
|
|
14201
14823
|
},
|
|
14202
|
-
cursor: {
|
|
14203
|
-
type: "string",
|
|
14204
|
-
description: "Pagination cursor"
|
|
14205
|
-
},
|
|
14206
|
-
before: {
|
|
14207
|
-
type: "string",
|
|
14208
|
-
description: "Only conversations or messages before this ISO timestamp"
|
|
14209
|
-
},
|
|
14210
|
-
after: {
|
|
14211
|
-
type: "string",
|
|
14212
|
-
description: "Only conversations or messages after this ISO timestamp"
|
|
14213
|
-
},
|
|
14214
14824
|
limit: {
|
|
14215
14825
|
type: "string",
|
|
14216
|
-
description: "Rows to return
|
|
14217
|
-
},
|
|
14218
|
-
header: {
|
|
14219
|
-
type: "string",
|
|
14220
|
-
alias: "H",
|
|
14221
|
-
description: "Request header for the UseSocial proxy, as Name: value"
|
|
14826
|
+
description: "Rows to return"
|
|
14222
14827
|
}
|
|
14223
14828
|
},
|
|
14224
|
-
run: async ({ args
|
|
14829
|
+
run: async ({ args }) => {
|
|
14225
14830
|
const parsedArgs = args;
|
|
14226
14831
|
const positionals = Array.isArray(args._) ? args._.map(String) : [];
|
|
14227
14832
|
if (positionals[1] === "mark") {
|
|
@@ -14238,52 +14843,66 @@ const buildMessages$1 = (deps, listCommand, readCommand) => {
|
|
|
14238
14843
|
});
|
|
14239
14844
|
return;
|
|
14240
14845
|
}
|
|
14241
|
-
const target = typeof args.target === "string" ? args.target : Array.isArray(args._) && typeof args._[0] === "string" ? args._[0] : void 0;
|
|
14242
|
-
|
|
14243
|
-
|
|
14244
|
-
|
|
14245
|
-
|
|
14246
|
-
|
|
14247
|
-
|
|
14248
|
-
});
|
|
14249
|
-
return;
|
|
14250
|
-
}
|
|
14251
|
-
rejectMessagesArgs("linkedin messages", parsedArgs, readArgNames);
|
|
14252
|
-
const account = typeof args.account === "string" ? args.account : void 0;
|
|
14253
|
-
const classified = await deps.classifyMessageTarget("target", target, {
|
|
14254
|
-
platform: "linkedin",
|
|
14255
|
-
account
|
|
14256
|
-
});
|
|
14257
|
-
if (classified.kind === "person") {
|
|
14258
|
-
const chatId = await resolveProfileChatId({
|
|
14259
|
-
account,
|
|
14260
|
-
deps,
|
|
14261
|
-
commandHeaderInput: args.header,
|
|
14262
|
-
rawArgs,
|
|
14263
|
-
profileId: classified.id,
|
|
14264
|
-
target
|
|
14265
|
-
});
|
|
14266
|
-
await readCommand.run?.({
|
|
14267
|
-
rawArgs,
|
|
14268
|
-
args: {
|
|
14269
|
-
...pickedArgs(parsedArgs, readArgNames),
|
|
14270
|
-
target: `chat_id:${chatId}`
|
|
14271
|
-
},
|
|
14272
|
-
cmd: readCommand
|
|
14273
|
-
});
|
|
14274
|
-
return;
|
|
14275
|
-
}
|
|
14276
|
-
await readCommand.run?.({
|
|
14277
|
-
rawArgs,
|
|
14278
|
-
args: {
|
|
14279
|
-
...pickedArgs(parsedArgs, readArgNames),
|
|
14280
|
-
target
|
|
14281
|
-
},
|
|
14282
|
-
cmd: readCommand
|
|
14283
|
-
});
|
|
14846
|
+
const target = typeof args.target === "string" ? args.target : Array.isArray(args._) && typeof args._[0] === "string" ? args._[0] : void 0;
|
|
14847
|
+
rejectMessagesArgs("linkedin messages", parsedArgs, cacheReadArgNames);
|
|
14848
|
+
const chatId = target === void 0 ? void 0 : chatIdFromCachedMessageTarget(deps, target);
|
|
14849
|
+
await withFreshLinkedinCache(deps, parsedArgs, "li_messages", "messages", (cache) => {
|
|
14850
|
+
const where = chatId === void 0 ? "" : ` WHERE m.chat_id = ${sqlString(chatId)}`;
|
|
14851
|
+
deps.printData(cache.query(`SELECT m.*, p.handle AS sender_handle, p.display_name AS sender_name, p.avatar_url AS sender_avatar_url, p.headline AS sender_headline FROM li_messages m LEFT JOIN li_profiles p ON m.sender_id = p.provider_id${where} ORDER BY m.timestamp DESC${limitClause(deps, args.limit)}`));
|
|
14852
|
+
}, true);
|
|
14284
14853
|
}
|
|
14285
14854
|
});
|
|
14286
14855
|
};
|
|
14856
|
+
const buildConnections = (deps, baseCommand) => defineCommand({
|
|
14857
|
+
meta: commandMeta$2({
|
|
14858
|
+
name: "connections",
|
|
14859
|
+
description: "List cached LinkedIn connections [read] | usage: `social linkedin connections --limit N`",
|
|
14860
|
+
method: "GET",
|
|
14861
|
+
capability: "read",
|
|
14862
|
+
mutates: false,
|
|
14863
|
+
schemaContract: cacheReadSchemaContract(baseCommand)
|
|
14864
|
+
}),
|
|
14865
|
+
args: {
|
|
14866
|
+
account: {
|
|
14867
|
+
type: "string",
|
|
14868
|
+
description: "Account selector: @handle or profile_id:<id>; overrides config default"
|
|
14869
|
+
},
|
|
14870
|
+
limit: {
|
|
14871
|
+
type: "string",
|
|
14872
|
+
description: "Rows to return"
|
|
14873
|
+
}
|
|
14874
|
+
},
|
|
14875
|
+
run: async ({ args }) => {
|
|
14876
|
+
const parsedArgs = args;
|
|
14877
|
+
rejectUnsupportedArgs("linkedin connections", parsedArgs, cacheReadArgNames);
|
|
14878
|
+
await readConnections(deps, parsedArgs);
|
|
14879
|
+
}
|
|
14880
|
+
});
|
|
14881
|
+
const buildRequestsRead = (deps, baseCommand, requestType) => defineCommand({
|
|
14882
|
+
meta: commandMeta$2({
|
|
14883
|
+
name: requestType,
|
|
14884
|
+
description: `List cached ${requestType} LinkedIn connection requests [read] | usage: \`social linkedin requests ${requestType} --limit N\``,
|
|
14885
|
+
method: "GET",
|
|
14886
|
+
capability: "read",
|
|
14887
|
+
mutates: false,
|
|
14888
|
+
schemaContract: cacheReadSchemaContract(baseCommand)
|
|
14889
|
+
}),
|
|
14890
|
+
args: {
|
|
14891
|
+
account: {
|
|
14892
|
+
type: "string",
|
|
14893
|
+
description: "Account selector: @handle or profile_id:<id>; overrides config default"
|
|
14894
|
+
},
|
|
14895
|
+
limit: {
|
|
14896
|
+
type: "string",
|
|
14897
|
+
description: "Rows to return"
|
|
14898
|
+
}
|
|
14899
|
+
},
|
|
14900
|
+
run: async ({ args }) => {
|
|
14901
|
+
const parsedArgs = args;
|
|
14902
|
+
rejectUnsupportedArgs(`linkedin requests ${requestType}`, parsedArgs, cacheReadArgNames);
|
|
14903
|
+
await readRequests(deps, parsedArgs, requestType);
|
|
14904
|
+
}
|
|
14905
|
+
});
|
|
14287
14906
|
const createLinkedinCommands = (deps) => {
|
|
14288
14907
|
const built = LINKEDIN_OPERATIONS.map((op) => ({
|
|
14289
14908
|
op,
|
|
@@ -14317,7 +14936,13 @@ const createLinkedinCommands = (deps) => {
|
|
|
14317
14936
|
subCommands[entry.op.config.command] = entry.command;
|
|
14318
14937
|
}
|
|
14319
14938
|
subCommands.message = buildMessage$1(deps, find("MessagesMessage"));
|
|
14320
|
-
subCommands.messages = buildMessages$1(deps, find("MessagesList")
|
|
14939
|
+
subCommands.messages = buildMessages$1(deps, find("MessagesList"));
|
|
14940
|
+
subCommands.connections = buildConnections(deps, find("UserConnections"));
|
|
14941
|
+
subCommands.posts = buildPosts(deps, find("UserPosts"));
|
|
14942
|
+
const requestsCommands = subCommands.requests?.subCommands;
|
|
14943
|
+
if (requestsCommands === void 0) throw new Error("Missing LinkedIn requests command group");
|
|
14944
|
+
requestsCommands.received = buildRequestsRead(deps, find("RequestsReceived"), "received");
|
|
14945
|
+
requestsCommands.sent = buildRequestsRead(deps, find("RequestsSent"), "sent");
|
|
14321
14946
|
subCommands.sync = buildSync$1(deps, "linkedin");
|
|
14322
14947
|
subCommands.sql = buildSql$1(deps, "linkedin");
|
|
14323
14948
|
return defineCommand({
|
|
@@ -14579,6 +15204,70 @@ const { costBPSForCategory } = createPricing([
|
|
|
14579
15204
|
"featureId": "counts_all_request"
|
|
14580
15205
|
}
|
|
14581
15206
|
]);
|
|
15207
|
+
//#endregion
|
|
15208
|
+
//#region ../../packages/x/src/profiles.ts
|
|
15209
|
+
const XUser = looseObject({
|
|
15210
|
+
id: string(),
|
|
15211
|
+
username: string().optional(),
|
|
15212
|
+
name: string().optional(),
|
|
15213
|
+
profile_image_url: string().nullish(),
|
|
15214
|
+
description: string().nullish()
|
|
15215
|
+
});
|
|
15216
|
+
const rowFrom = (user) => {
|
|
15217
|
+
const providerId = user.id.trim();
|
|
15218
|
+
if (!providerId) return null;
|
|
15219
|
+
const handle = user.username?.trim() || null;
|
|
15220
|
+
const displayName = user.name?.trim() || handle || "";
|
|
15221
|
+
if (!displayName) return null;
|
|
15222
|
+
return {
|
|
15223
|
+
providerId,
|
|
15224
|
+
handle,
|
|
15225
|
+
displayName,
|
|
15226
|
+
avatarURL: user.profile_image_url?.trim() || null,
|
|
15227
|
+
headline: user.description?.trim() || null,
|
|
15228
|
+
raw: user
|
|
15229
|
+
};
|
|
15230
|
+
};
|
|
15231
|
+
const SingleBody = looseObject({ data: XUser });
|
|
15232
|
+
const extractProfile = (body) => {
|
|
15233
|
+
const parsed = SingleBody.safeParse(body);
|
|
15234
|
+
if (!parsed.success) return [];
|
|
15235
|
+
const row = rowFrom(parsed.data.data);
|
|
15236
|
+
return row ? [row] : [];
|
|
15237
|
+
};
|
|
15238
|
+
const ListBody = looseObject({ data: array(unknown()) });
|
|
15239
|
+
const extractList = (body) => {
|
|
15240
|
+
const parsed = ListBody.safeParse(body);
|
|
15241
|
+
if (!parsed.success) return [];
|
|
15242
|
+
return parsed.data.data.flatMap((item) => {
|
|
15243
|
+
const user = XUser.safeParse(item);
|
|
15244
|
+
if (!user.success) return [];
|
|
15245
|
+
const row = rowFrom(user.data);
|
|
15246
|
+
return row ? [row] : [];
|
|
15247
|
+
});
|
|
15248
|
+
};
|
|
15249
|
+
const IncludesUsersBody = looseObject({ includes: looseObject({ users: array(unknown()).optional() }).optional() });
|
|
15250
|
+
const extractIncludesUsers = (body) => {
|
|
15251
|
+
const parsed = IncludesUsersBody.safeParse(body);
|
|
15252
|
+
if (!parsed.success) return [];
|
|
15253
|
+
return (parsed.data.includes?.users ?? []).flatMap((item) => {
|
|
15254
|
+
const user = XUser.safeParse(item);
|
|
15255
|
+
if (!user.success) return [];
|
|
15256
|
+
const row = rowFrom(user.data);
|
|
15257
|
+
return row ? [row] : [];
|
|
15258
|
+
});
|
|
15259
|
+
};
|
|
15260
|
+
const xProfileExtractors = {
|
|
15261
|
+
"GET /dm_conversations/:id/dm_events": extractIncludesUsers,
|
|
15262
|
+
"GET /dm_events": extractIncludesUsers,
|
|
15263
|
+
"GET /lists/:id/members": extractList,
|
|
15264
|
+
"GET /tweets/:id/liking_users": extractList,
|
|
15265
|
+
"GET /tweets/:id/retweeted_by": extractList,
|
|
15266
|
+
"GET /users/:id": extractProfile,
|
|
15267
|
+
"GET /users/:id/followers": extractList,
|
|
15268
|
+
"GET /users/:id/following": extractList,
|
|
15269
|
+
"GET /users/by/username/:username": extractProfile
|
|
15270
|
+
};
|
|
14582
15271
|
const X_OPERATIONS = [
|
|
14583
15272
|
{
|
|
14584
15273
|
schemaBaseName: "TweetsDelete",
|
|
@@ -17314,14 +18003,14 @@ const buildArgs = (op) => {
|
|
|
17314
18003
|
};
|
|
17315
18004
|
return args;
|
|
17316
18005
|
};
|
|
17317
|
-
const camelCaseArg = (name) => name.replace(/[-_.]+([A-Za-z0-9])/g, (_match, next) => next.toUpperCase());
|
|
17318
|
-
const snakeCaseArg = (name) => name.replace(/[-.]+/g, "_");
|
|
17319
|
-
const argAliases = (name) => {
|
|
18006
|
+
const camelCaseArg$1 = (name) => name.replace(/[-_.]+([A-Za-z0-9])/g, (_match, next) => next.toUpperCase());
|
|
18007
|
+
const snakeCaseArg$1 = (name) => name.replace(/[-.]+/g, "_");
|
|
18008
|
+
const argAliases$1 = (name) => {
|
|
17320
18009
|
const aliases = new Set([
|
|
17321
18010
|
name,
|
|
17322
|
-
camelCaseArg(name),
|
|
18011
|
+
camelCaseArg$1(name),
|
|
17323
18012
|
flagName(name),
|
|
17324
|
-
snakeCaseArg(name)
|
|
18013
|
+
snakeCaseArg$1(name)
|
|
17325
18014
|
]);
|
|
17326
18015
|
if (name.startsWith("no-")) aliases.add(name.slice(3));
|
|
17327
18016
|
if (name === "header") aliases.add("H");
|
|
@@ -17329,7 +18018,7 @@ const argAliases = (name) => {
|
|
|
17329
18018
|
};
|
|
17330
18019
|
const knownArgNamesFor = (op) => {
|
|
17331
18020
|
const names = new Set(["_"]);
|
|
17332
|
-
for (const name of Object.keys(buildArgs(op))) for (const alias of argAliases(name)) names.add(alias);
|
|
18021
|
+
for (const name of Object.keys(buildArgs(op))) for (const alias of argAliases$1(name)) names.add(alias);
|
|
17333
18022
|
return names;
|
|
17334
18023
|
};
|
|
17335
18024
|
const unsupportedFlagError = (name, value) => {
|
|
@@ -17340,7 +18029,7 @@ const unsupportedFlagError = (name, value) => {
|
|
|
17340
18029
|
message: `${flag} is not supported by this command.`
|
|
17341
18030
|
}]);
|
|
17342
18031
|
};
|
|
17343
|
-
const rejectUnknownArgs = (op, args) => {
|
|
18032
|
+
const rejectUnknownArgs$1 = (op, args) => {
|
|
17344
18033
|
const known = knownArgNamesFor(op);
|
|
17345
18034
|
for (const [name, value] of Object.entries(args)) if (!known.has(name)) throw unsupportedFlagError(name, value);
|
|
17346
18035
|
};
|
|
@@ -17369,6 +18058,7 @@ const parseQueryValue = (deps, param, value, displayName) => {
|
|
|
17369
18058
|
}));
|
|
17370
18059
|
return deps.parseQueryString(displayName, value);
|
|
17371
18060
|
};
|
|
18061
|
+
const mergePresetQueryValue = (preset, value) => Array.from(new Set(`${preset},${value}`.split(",").map((part) => part.trim()).filter(Boolean))).join(",");
|
|
17372
18062
|
const setQueryParams = (deps, op, input, params) => {
|
|
17373
18063
|
const positionalQuery = new Set(op.config.positionalQueryParams ?? []);
|
|
17374
18064
|
for (const param of op.parameters.filter((p) => p.in === "query")) {
|
|
@@ -17384,8 +18074,10 @@ const setQueryParams = (deps, op, input, params) => {
|
|
|
17384
18074
|
}
|
|
17385
18075
|
const displayName = positionalQuery.has(param.name) ? flagName(param.name) : `--${cliFlagNameFor(param.name)}`;
|
|
17386
18076
|
const preset = op.config.fieldPresets?.[param.name];
|
|
17387
|
-
if (value !== void 0)
|
|
17388
|
-
|
|
18077
|
+
if (value !== void 0) {
|
|
18078
|
+
const parsedValue = parseQueryValue(deps, param, value, displayName);
|
|
18079
|
+
params.set(param.name, preset === void 0 ? parsedValue : mergePresetQueryValue(deps.parseQueryString(displayName, preset), parsedValue));
|
|
18080
|
+
} else if (preset !== void 0) params.set(param.name, deps.parseQueryString(displayName, preset));
|
|
17389
18081
|
}
|
|
17390
18082
|
};
|
|
17391
18083
|
const needsOwnAccount = (op) => op.parameters.some((p) => p.in === "path" && ownMode(op, p.name) !== void 0);
|
|
@@ -17456,7 +18148,7 @@ const resolveBodyTextFromStdin = async (op, deps, args, commandLabel) => {
|
|
|
17456
18148
|
else if (required) throw noBodyTextError$1(commandLabel);
|
|
17457
18149
|
};
|
|
17458
18150
|
const runOperation = async (op, contract, deps, args, rawArgs = []) => {
|
|
17459
|
-
rejectUnknownArgs(op, args);
|
|
18151
|
+
rejectUnknownArgs$1(op, args);
|
|
17460
18152
|
await resolveBodyTextFromStdin(op, deps, args, [
|
|
17461
18153
|
"social",
|
|
17462
18154
|
"x",
|
|
@@ -17591,28 +18283,69 @@ const defineOperationCommand = (op, deps) => {
|
|
|
17591
18283
|
//#region ../../packages/x/src/cli/commands.ts
|
|
17592
18284
|
const hiddenRootCommands = new Set(["search"]);
|
|
17593
18285
|
const isObjectRecord = (value) => typeof value === "object" && value !== null && !Array.isArray(value);
|
|
17594
|
-
const confirmTTY = async (message) => {
|
|
17595
|
-
const readline = createInterface({
|
|
17596
|
-
input: process.stdin,
|
|
17597
|
-
output: process.stdout
|
|
17598
|
-
});
|
|
17599
|
-
try {
|
|
17600
|
-
const answer = await readline.question(`${message} [y/N] `);
|
|
17601
|
-
return /^(y|yes)$/i.test(answer.trim());
|
|
17602
|
-
} finally {
|
|
17603
|
-
readline.close();
|
|
17604
|
-
}
|
|
17605
|
-
};
|
|
17606
18286
|
const requireAccountSelector = (selector) => {
|
|
17607
18287
|
if (selector === void 0) throw new UsageError("Selected X account is missing an account selector.");
|
|
17608
18288
|
return selector;
|
|
17609
18289
|
};
|
|
18290
|
+
const xSyncCountFields = {
|
|
18291
|
+
x_followers: "followers_count",
|
|
18292
|
+
x_following: "following_count",
|
|
18293
|
+
x_tweets: "tweet_count"
|
|
18294
|
+
};
|
|
18295
|
+
const ownMetricCount = async (args) => {
|
|
18296
|
+
const field = xSyncCountFields[args.collectionKey];
|
|
18297
|
+
if (field === void 0) return null;
|
|
18298
|
+
const params = new URLSearchParams({ "user.fields": "public_metrics" });
|
|
18299
|
+
const body = await (await args.deps.api.get(`x/users/${encodeURIComponent(args.ownId)}?${params.toString()}`, { headers: args.accountHeaders })).json();
|
|
18300
|
+
const data = isObjectRecord(body) ? body.data : void 0;
|
|
18301
|
+
const publicMetrics = isObjectRecord(data) ? data.public_metrics : void 0;
|
|
18302
|
+
const count = isObjectRecord(publicMetrics) ? publicMetrics[field] : void 0;
|
|
18303
|
+
return typeof count === "number" && Number.isFinite(count) ? count : null;
|
|
18304
|
+
};
|
|
18305
|
+
const syncQuoteLine = (estCredits, quoteLine) => `Estimated ~${Math.ceil(estCredits)} credits.\n${quoteLine}\n`;
|
|
18306
|
+
const autoCapPartialLine = () => `Hit the ${AUTO_CAP_CREDITS} credit auto-cap (partial). Re-run with --credits <N> to continue.\n`;
|
|
17610
18307
|
const buildSync = (deps, platform) => defineCommand({
|
|
17611
|
-
meta: {
|
|
18308
|
+
meta: commandMeta$1({
|
|
17612
18309
|
name: "sync",
|
|
17613
18310
|
description: "Sync a collection into the local cache (spends credits)",
|
|
17614
|
-
|
|
17615
|
-
|
|
18311
|
+
method: "GET",
|
|
18312
|
+
capability: "read",
|
|
18313
|
+
mutates: true,
|
|
18314
|
+
contract: {
|
|
18315
|
+
auth: {
|
|
18316
|
+
required: true,
|
|
18317
|
+
scopes: ["read"]
|
|
18318
|
+
},
|
|
18319
|
+
capability: "read",
|
|
18320
|
+
confirmation: {
|
|
18321
|
+
required: true,
|
|
18322
|
+
recommended: true,
|
|
18323
|
+
description: "Sync can spend usage credits and writes local cache state."
|
|
18324
|
+
},
|
|
18325
|
+
idempotency: {
|
|
18326
|
+
kind: "non-idempotent",
|
|
18327
|
+
description: "Repeated syncs can spend additional usage credits and update local rows."
|
|
18328
|
+
},
|
|
18329
|
+
method: "GET",
|
|
18330
|
+
mutates: true,
|
|
18331
|
+
outputSchema: unknown(),
|
|
18332
|
+
pagination: { style: "none" },
|
|
18333
|
+
responseShaping: { supported: false },
|
|
18334
|
+
examples: [{ command: "social x sync followers --credits 1000" }],
|
|
18335
|
+
cost: {
|
|
18336
|
+
metered: true,
|
|
18337
|
+
unit: "usage_credit",
|
|
18338
|
+
notes: ["Upstream reads during sync are metered."]
|
|
18339
|
+
},
|
|
18340
|
+
hazards: [{
|
|
18341
|
+
code: "local_cache_write",
|
|
18342
|
+
message: "Writes synced X rows and cursor state to the local cache."
|
|
18343
|
+
}, {
|
|
18344
|
+
code: "usage_credit_spend",
|
|
18345
|
+
message: "May spend usage credits while reading upstream X data."
|
|
18346
|
+
}]
|
|
18347
|
+
}
|
|
18348
|
+
}),
|
|
17616
18349
|
args: {
|
|
17617
18350
|
collection: {
|
|
17618
18351
|
type: "positional",
|
|
@@ -17623,15 +18356,13 @@ const buildSync = (deps, platform) => defineCommand({
|
|
|
17623
18356
|
type: "string",
|
|
17624
18357
|
description: "ISO date/datetime or compact duration like 2:days|3:weeks|5:hours"
|
|
17625
18358
|
},
|
|
17626
|
-
|
|
18359
|
+
credits: {
|
|
17627
18360
|
type: "string",
|
|
17628
|
-
|
|
17629
|
-
description: "Hard cap on spend in USD (default 10)"
|
|
18361
|
+
description: "Consent and hard cap in credits"
|
|
17630
18362
|
},
|
|
17631
|
-
|
|
18363
|
+
reset: {
|
|
17632
18364
|
type: "boolean",
|
|
17633
|
-
|
|
17634
|
-
description: "Skip the TTY confirmation"
|
|
18365
|
+
description: "Delete local rows and sync state for the selected collection"
|
|
17635
18366
|
},
|
|
17636
18367
|
account: {
|
|
17637
18368
|
type: "string",
|
|
@@ -17639,7 +18370,7 @@ const buildSync = (deps, platform) => defineCommand({
|
|
|
17639
18370
|
}
|
|
17640
18371
|
},
|
|
17641
18372
|
run: async ({ args }) => {
|
|
17642
|
-
const collections =
|
|
18373
|
+
const collections = syncableCollectionsFor(platform);
|
|
17643
18374
|
const shortName = (collection) => collection.key.replace(/^(x|li)_/, "");
|
|
17644
18375
|
const account = typeof args.account === "string" ? args.account : void 0;
|
|
17645
18376
|
const res = await deps.resolveOwnXAccount({ account });
|
|
@@ -17647,6 +18378,7 @@ const buildSync = (deps, platform) => defineCommand({
|
|
|
17647
18378
|
const accountSelector = requireAccountSelector(res.account.selector);
|
|
17648
18379
|
const accountHeaders = { "x-social-account-id": accountSelector };
|
|
17649
18380
|
if (!args.collection) {
|
|
18381
|
+
if (args.reset === true) throw new UsageError("--reset requires a collection");
|
|
17650
18382
|
const cache = openCache({
|
|
17651
18383
|
platform,
|
|
17652
18384
|
accountId: accountSelector
|
|
@@ -17667,24 +18399,63 @@ const buildSync = (deps, platform) => defineCommand({
|
|
|
17667
18399
|
}
|
|
17668
18400
|
const collection = collections.find((candidate) => shortName(candidate) === args.collection || candidate.key === args.collection);
|
|
17669
18401
|
if (!collection) throw new UsageError(`Unknown collection "${args.collection}". Available: ${collections.map(shortName).join(", ")}`);
|
|
18402
|
+
if (args.reset === true) {
|
|
18403
|
+
if (args.credits != null) throw new UsageError("--reset cannot be combined with --credits");
|
|
18404
|
+
if (args.since != null) throw new UsageError("--reset cannot be combined with --since");
|
|
18405
|
+
const cache = openCache({
|
|
18406
|
+
platform,
|
|
18407
|
+
accountId: accountSelector
|
|
18408
|
+
});
|
|
18409
|
+
const resetCollections = resetTablesFor(collection);
|
|
18410
|
+
try {
|
|
18411
|
+
deps.printData({
|
|
18412
|
+
collection: collection.key,
|
|
18413
|
+
reset: true,
|
|
18414
|
+
tables: cache.reset({
|
|
18415
|
+
tables: resetCollections.map((resetCollection) => resetCollection.key),
|
|
18416
|
+
collections: resetCollections.map((resetCollection) => resetCollection.key)
|
|
18417
|
+
})
|
|
18418
|
+
});
|
|
18419
|
+
} finally {
|
|
18420
|
+
cache.close();
|
|
18421
|
+
}
|
|
18422
|
+
return;
|
|
18423
|
+
}
|
|
17670
18424
|
let since;
|
|
17671
18425
|
if (args.since != null) {
|
|
17672
18426
|
if (!collection.sinceField) throw new UsageError(`${shortName(collection)} does not support --since`);
|
|
17673
18427
|
since = parseSince(String(args.since));
|
|
17674
18428
|
}
|
|
17675
|
-
const
|
|
17676
|
-
if (!Number.isFinite(
|
|
17677
|
-
const billingLimitCredits = usdToCredits(limitUSD);
|
|
18429
|
+
const creditsArg = args.credits != null && args.credits !== "" ? Number(args.credits) : null;
|
|
18430
|
+
if (creditsArg != null && (!Number.isFinite(creditsArg) || creditsArg < 0)) throw new UsageError("--credits must be a non-negative number of credits");
|
|
17678
18431
|
const cache = openCache({
|
|
17679
18432
|
platform,
|
|
17680
18433
|
accountId: accountSelector
|
|
17681
18434
|
});
|
|
17682
|
-
const
|
|
17683
|
-
|
|
17684
|
-
|
|
17685
|
-
|
|
17686
|
-
|
|
17687
|
-
|
|
18435
|
+
const firstSync = cache.getCursor(collection.key)?.lastSynced == null;
|
|
18436
|
+
const count = firstSync ? await ownMetricCount({
|
|
18437
|
+
deps,
|
|
18438
|
+
collectionKey: collection.key,
|
|
18439
|
+
ownId,
|
|
18440
|
+
accountHeaders
|
|
18441
|
+
}) : null;
|
|
18442
|
+
const plan = planSync({
|
|
18443
|
+
collection,
|
|
18444
|
+
creditsArg,
|
|
18445
|
+
count,
|
|
18446
|
+
firstSync
|
|
18447
|
+
});
|
|
18448
|
+
if (plan.mode === "quote") {
|
|
18449
|
+
cache.close();
|
|
18450
|
+
process.stderr.write(syncQuoteLine(plan.estCredits, plan.quoteLine));
|
|
18451
|
+
return;
|
|
18452
|
+
}
|
|
18453
|
+
if (plan.mode === "reject") {
|
|
18454
|
+
cache.close();
|
|
18455
|
+
throw new UsageError(`${shortName(collection)}: ${plan.message}`);
|
|
18456
|
+
}
|
|
18457
|
+
const onEstimate = (estimate) => {
|
|
18458
|
+
const line = estimate.unknown ? `Estimated ~${estimate.perPageCredits} credits/page; total unknown, bounded by ${estimate.limitCredits} credits.` : `Estimated ~${estimate.estCredits} credits across ${estimate.estPages} page(s); limit ${estimate.limitCredits} credits.`;
|
|
17688
18459
|
process.stderr.write(`${line}\n`);
|
|
17689
18460
|
return true;
|
|
17690
18461
|
};
|
|
@@ -17695,14 +18466,17 @@ const buildSync = (deps, platform) => defineCommand({
|
|
|
17695
18466
|
cache,
|
|
17696
18467
|
platform,
|
|
17697
18468
|
ownId,
|
|
17698
|
-
accountHeaders
|
|
18469
|
+
accountHeaders,
|
|
18470
|
+
profileExtractors: xProfileExtractors
|
|
17699
18471
|
}, {
|
|
17700
18472
|
collection,
|
|
17701
18473
|
since,
|
|
17702
|
-
|
|
18474
|
+
expectedTotalCount: firstSync ? count : null,
|
|
18475
|
+
billingLimitCredits: plan.billingLimitCredits,
|
|
17703
18476
|
onEstimate,
|
|
17704
18477
|
onProgress
|
|
17705
18478
|
});
|
|
18479
|
+
if (plan.capAndGo && summary.stoppedReason === "credit-limit") process.stderr.write(autoCapPartialLine());
|
|
17706
18480
|
deps.printData(summary);
|
|
17707
18481
|
} finally {
|
|
17708
18482
|
cache.close();
|
|
@@ -17753,92 +18527,6 @@ const buildSql = (deps, platform) => defineCommand({
|
|
|
17753
18527
|
}
|
|
17754
18528
|
}
|
|
17755
18529
|
});
|
|
17756
|
-
const defaultDMEventFields = [
|
|
17757
|
-
"attachments",
|
|
17758
|
-
"created_at",
|
|
17759
|
-
"dm_conversation_id",
|
|
17760
|
-
"entities",
|
|
17761
|
-
"event_type",
|
|
17762
|
-
"id",
|
|
17763
|
-
"participant_ids",
|
|
17764
|
-
"referenced_tweets",
|
|
17765
|
-
"sender_id",
|
|
17766
|
-
"text"
|
|
17767
|
-
].join(",");
|
|
17768
|
-
const defaultDMExpansions = [
|
|
17769
|
-
"attachments.media_keys",
|
|
17770
|
-
"participant_ids",
|
|
17771
|
-
"referenced_tweets.id",
|
|
17772
|
-
"sender_id"
|
|
17773
|
-
].join(",");
|
|
17774
|
-
const defaultDMUserFields = [
|
|
17775
|
-
"affiliation",
|
|
17776
|
-
"connection_status",
|
|
17777
|
-
"created_at",
|
|
17778
|
-
"description",
|
|
17779
|
-
"entities",
|
|
17780
|
-
"id",
|
|
17781
|
-
"location",
|
|
17782
|
-
"most_recent_tweet_id",
|
|
17783
|
-
"name",
|
|
17784
|
-
"pinned_tweet_id",
|
|
17785
|
-
"profile_banner_url",
|
|
17786
|
-
"profile_image_url",
|
|
17787
|
-
"protected",
|
|
17788
|
-
"public_metrics",
|
|
17789
|
-
"receives_your_dm",
|
|
17790
|
-
"subscription_type",
|
|
17791
|
-
"url",
|
|
17792
|
-
"username",
|
|
17793
|
-
"verified",
|
|
17794
|
-
"verified_type",
|
|
17795
|
-
"withheld"
|
|
17796
|
-
].join(",");
|
|
17797
|
-
const defaultDMTweetFields = [
|
|
17798
|
-
"article",
|
|
17799
|
-
"attachments",
|
|
17800
|
-
"author_id",
|
|
17801
|
-
"card_uri",
|
|
17802
|
-
"community_id",
|
|
17803
|
-
"context_annotations",
|
|
17804
|
-
"conversation_id",
|
|
17805
|
-
"created_at",
|
|
17806
|
-
"display_text_range",
|
|
17807
|
-
"edit_controls",
|
|
17808
|
-
"edit_history_tweet_ids",
|
|
17809
|
-
"entities",
|
|
17810
|
-
"geo",
|
|
17811
|
-
"id",
|
|
17812
|
-
"in_reply_to_user_id",
|
|
17813
|
-
"lang",
|
|
17814
|
-
"matched_media_notes",
|
|
17815
|
-
"media_metadata",
|
|
17816
|
-
"note_request_suggestions",
|
|
17817
|
-
"note_tweet",
|
|
17818
|
-
"paid_partnership",
|
|
17819
|
-
"possibly_sensitive",
|
|
17820
|
-
"public_metrics",
|
|
17821
|
-
"referenced_tweets",
|
|
17822
|
-
"reply_settings",
|
|
17823
|
-
"scopes",
|
|
17824
|
-
"source",
|
|
17825
|
-
"suggested_source_links",
|
|
17826
|
-
"suggested_source_links_with_counts",
|
|
17827
|
-
"text",
|
|
17828
|
-
"withheld"
|
|
17829
|
-
].join(",");
|
|
17830
|
-
const defaultDMMediaFields = [
|
|
17831
|
-
"alt_text",
|
|
17832
|
-
"duration_ms",
|
|
17833
|
-
"height",
|
|
17834
|
-
"media_key",
|
|
17835
|
-
"preview_image_url",
|
|
17836
|
-
"public_metrics",
|
|
17837
|
-
"type",
|
|
17838
|
-
"url",
|
|
17839
|
-
"variants",
|
|
17840
|
-
"width"
|
|
17841
|
-
].join(",");
|
|
17842
18530
|
const messagesOp = {
|
|
17843
18531
|
schemaBaseName: "Messages",
|
|
17844
18532
|
hasBody: false,
|
|
@@ -17888,114 +18576,197 @@ const messageOp = {
|
|
|
17888
18576
|
},
|
|
17889
18577
|
parameters: []
|
|
17890
18578
|
};
|
|
17891
|
-
const
|
|
17892
|
-
const
|
|
18579
|
+
const operationForCommand = (command) => {
|
|
18580
|
+
const op = X_OPERATIONS.find((candidate) => candidate.config.command === command);
|
|
18581
|
+
if (op === void 0) throw new Error(`x_cli_operation_missing:${command}`);
|
|
18582
|
+
return op;
|
|
18583
|
+
};
|
|
18584
|
+
const cacheReadArgs = () => ({
|
|
18585
|
+
limit: {
|
|
18586
|
+
type: "string",
|
|
18587
|
+
description: "Rows to return"
|
|
18588
|
+
},
|
|
18589
|
+
account: {
|
|
18590
|
+
type: "string",
|
|
18591
|
+
description: "Account selector: @handle or profile_id:<id>; overrides config default"
|
|
18592
|
+
}
|
|
18593
|
+
});
|
|
18594
|
+
const cacheReadContract = (op, commandName) => {
|
|
18595
|
+
const { inputSchema: _inputSchema, ...contract } = contractFor$1(op, commandName, "");
|
|
18596
|
+
return {
|
|
18597
|
+
...contract,
|
|
18598
|
+
pagination: { style: "none" }
|
|
18599
|
+
};
|
|
18600
|
+
};
|
|
18601
|
+
const camelCaseArg = (name) => name.replace(/[-_.]+([A-Za-z0-9])/g, (_match, next) => next.toUpperCase());
|
|
18602
|
+
const snakeCaseArg = (name) => name.replace(/[-.]+/g, "_");
|
|
18603
|
+
const flagNameForArg = (name) => name.replace(/[_.]+/g, "-");
|
|
18604
|
+
const argAliases = (name) => {
|
|
18605
|
+
const aliases = new Set([
|
|
18606
|
+
name,
|
|
18607
|
+
camelCaseArg(name),
|
|
18608
|
+
flagNameForArg(name),
|
|
18609
|
+
snakeCaseArg(name)
|
|
18610
|
+
]);
|
|
18611
|
+
if (name.startsWith("no-")) aliases.add(name.slice(3));
|
|
18612
|
+
if (name === "header") aliases.add("H");
|
|
18613
|
+
return aliases;
|
|
18614
|
+
};
|
|
18615
|
+
const rejectUnknownArgs = (args, argsDef) => {
|
|
18616
|
+
const known = new Set(["_"]);
|
|
18617
|
+
for (const name of Object.keys(argsDef)) for (const alias of argAliases(name)) known.add(alias);
|
|
18618
|
+
for (const [name, value] of Object.entries(args)) {
|
|
18619
|
+
if (value === void 0 || known.has(name)) continue;
|
|
18620
|
+
const flag = value === false ? `--no-${flagNameForArg(name)}` : `--${flagNameForArg(name)}`;
|
|
18621
|
+
throw new ZodError([{
|
|
18622
|
+
code: "custom",
|
|
18623
|
+
path: [name],
|
|
18624
|
+
message: `${flag} is not supported by this command.`
|
|
18625
|
+
}]);
|
|
18626
|
+
}
|
|
18627
|
+
};
|
|
18628
|
+
const collectionForCacheRead = (key) => {
|
|
18629
|
+
const collection = collectionByKey(key);
|
|
18630
|
+
if (collection === void 0) throw new Error(`x_cache_collection_missing:${key}`);
|
|
18631
|
+
return collection;
|
|
18632
|
+
};
|
|
18633
|
+
const parseOptionalLimit = (deps, value, max) => value === void 0 ? void 0 : deps.parseIntegerString("--limit", value, {
|
|
18634
|
+
min: 1,
|
|
18635
|
+
max
|
|
18636
|
+
});
|
|
18637
|
+
const appendLimit = (sql, limit) => limit === void 0 ? sql : `${sql} LIMIT ${limit}`;
|
|
18638
|
+
const withSender = (rows) => rows.map((row) => {
|
|
18639
|
+
const { sender_handle: senderHandle, sender_name: senderName, sender_avatar_url: senderAvatarURL, sender_headline: senderHeadline, ...message } = row;
|
|
18640
|
+
if (![
|
|
18641
|
+
senderHandle,
|
|
18642
|
+
senderName,
|
|
18643
|
+
senderAvatarURL,
|
|
18644
|
+
senderHeadline
|
|
18645
|
+
].some((value) => value !== null && value !== void 0)) return message;
|
|
18646
|
+
return {
|
|
18647
|
+
...message,
|
|
18648
|
+
sender: {
|
|
18649
|
+
...typeof message.sender_id === "string" ? { id: message.sender_id } : {},
|
|
18650
|
+
handle: senderHandle,
|
|
18651
|
+
name: senderName,
|
|
18652
|
+
avatar_url: senderAvatarURL,
|
|
18653
|
+
headline: senderHeadline
|
|
18654
|
+
}
|
|
18655
|
+
};
|
|
18656
|
+
});
|
|
18657
|
+
const buildCacheRead = (deps, config) => {
|
|
18658
|
+
const contract = cacheReadContract(config.operation, config.name);
|
|
17893
18659
|
return defineCommand({
|
|
17894
18660
|
meta: commandMeta$1({
|
|
17895
|
-
name:
|
|
17896
|
-
description:
|
|
18661
|
+
name: config.name,
|
|
18662
|
+
description: config.description,
|
|
17897
18663
|
method: "GET",
|
|
17898
18664
|
capability: "read",
|
|
17899
18665
|
mutates: false,
|
|
17900
18666
|
contract
|
|
17901
18667
|
}),
|
|
17902
|
-
args:
|
|
17903
|
-
target: {
|
|
17904
|
-
type: "positional",
|
|
17905
|
-
required: false,
|
|
17906
|
-
description: "Target conversation or participant: chat URL, chat_id:<id>, @handle, profile URL, or profile_id:<id>",
|
|
17907
|
-
valueHint: "target"
|
|
17908
|
-
},
|
|
17909
|
-
limit: {
|
|
17910
|
-
type: "string",
|
|
17911
|
-
description: "Rows to return (1-100)"
|
|
17912
|
-
},
|
|
17913
|
-
cursor: {
|
|
17914
|
-
type: "string",
|
|
17915
|
-
description: "Cursor from a previous response"
|
|
17916
|
-
},
|
|
17917
|
-
"event-types": {
|
|
17918
|
-
type: "string",
|
|
17919
|
-
description: "Comma-separated event types to include"
|
|
17920
|
-
},
|
|
17921
|
-
"dm-event-fields": {
|
|
17922
|
-
type: "string",
|
|
17923
|
-
description: "DM event fields to request"
|
|
17924
|
-
},
|
|
17925
|
-
expansions: {
|
|
17926
|
-
type: "string",
|
|
17927
|
-
description: "DM event expansions to request"
|
|
17928
|
-
},
|
|
17929
|
-
"user-fields": {
|
|
17930
|
-
type: "string",
|
|
17931
|
-
description: "User fields to request"
|
|
17932
|
-
},
|
|
17933
|
-
"tweet-fields": {
|
|
17934
|
-
type: "string",
|
|
17935
|
-
description: "Tweet fields to request"
|
|
17936
|
-
},
|
|
17937
|
-
"media-fields": {
|
|
17938
|
-
type: "string",
|
|
17939
|
-
description: "Media fields to request"
|
|
17940
|
-
},
|
|
17941
|
-
account: {
|
|
17942
|
-
type: "string",
|
|
17943
|
-
description: "Account selector: @handle or profile_id:<id>; overrides config default"
|
|
17944
|
-
}
|
|
17945
|
-
},
|
|
18668
|
+
args: cacheReadArgs(),
|
|
17946
18669
|
run: async ({ args }) => {
|
|
17947
|
-
const positional = Array.isArray(args._) ? args._ : [];
|
|
17948
|
-
const targetInput = args.target ?? (positional.length > 0 ? positional[0] : void 0);
|
|
17949
|
-
const resolved = [];
|
|
17950
|
-
const params = new URLSearchParams();
|
|
17951
|
-
if (args.limit !== void 0) params.set("max_results", String(deps.parseIntegerString("--limit", args.limit, {
|
|
17952
|
-
min: 1,
|
|
17953
|
-
max: 100
|
|
17954
|
-
})));
|
|
17955
|
-
if (args.cursor !== void 0) params.set("pagination_token", deps.parseQueryString("--cursor", args.cursor));
|
|
17956
|
-
if (args["event-types"] !== void 0) params.set("event_types", deps.parseQueryString("--event-types", args["event-types"]));
|
|
17957
|
-
params.set("dm_event.fields", deps.parseQueryString("--dm-event-fields", args["dm-event-fields"] ?? defaultDMEventFields));
|
|
17958
|
-
params.set("expansions", deps.parseQueryString("--expansions", args.expansions ?? defaultDMExpansions));
|
|
17959
|
-
params.set("user.fields", deps.parseQueryString("--user-fields", args["user-fields"] ?? defaultDMUserFields));
|
|
17960
|
-
params.set("tweet.fields", deps.parseQueryString("--tweet-fields", args["tweet-fields"] ?? defaultDMTweetFields));
|
|
17961
|
-
params.set("media.fields", deps.parseQueryString("--media-fields", args["media-fields"] ?? defaultDMMediaFields));
|
|
17962
|
-
const query = params.toString();
|
|
17963
18670
|
const account = typeof args.account === "string" ? args.account : void 0;
|
|
17964
|
-
const
|
|
18671
|
+
const res = await deps.resolveOwnXAccount({ account });
|
|
18672
|
+
const ownId = res.xUserId;
|
|
18673
|
+
const accountSelector = requireAccountSelector(res.account.selector);
|
|
18674
|
+
const accountHeaders = { "x-social-account-id": accountSelector };
|
|
18675
|
+
const cache = openCache({
|
|
17965
18676
|
platform: "x",
|
|
17966
|
-
|
|
18677
|
+
accountId: accountSelector
|
|
17967
18678
|
});
|
|
17968
|
-
const
|
|
17969
|
-
const
|
|
17970
|
-
|
|
18679
|
+
const collection = collectionForCacheRead(config.collectionKey);
|
|
18680
|
+
const syncDeps = {
|
|
18681
|
+
api: deps.api,
|
|
18682
|
+
cache,
|
|
17971
18683
|
platform: "x",
|
|
17972
|
-
|
|
17973
|
-
|
|
17974
|
-
|
|
17975
|
-
|
|
17976
|
-
|
|
17977
|
-
|
|
17978
|
-
|
|
17979
|
-
|
|
17980
|
-
|
|
17981
|
-
const
|
|
17982
|
-
|
|
17983
|
-
|
|
17984
|
-
|
|
17985
|
-
|
|
17986
|
-
for (const event of data.data) if (isObjectRecord(event) && typeof event.sender_id === "string") {
|
|
17987
|
-
const sender = usersById.get(event.sender_id);
|
|
17988
|
-
if (sender !== void 0) event.sender = sender;
|
|
17989
|
-
}
|
|
17990
|
-
}
|
|
18684
|
+
ownId,
|
|
18685
|
+
accountHeaders,
|
|
18686
|
+
profileExtractors: xProfileExtractors
|
|
18687
|
+
};
|
|
18688
|
+
try {
|
|
18689
|
+
if ((await ensureFreshCache(syncDeps, {
|
|
18690
|
+
collection,
|
|
18691
|
+
ttlSeconds: config.refreshEveryCall === true ? 0 : void 0
|
|
18692
|
+
})).status === "never-synced") throw new UsageError(`No synced ${config.name} yet. Run \`x sync ${config.name}\` first.`);
|
|
18693
|
+
const limit = parseOptionalLimit(deps, args.limit, config.limitMax);
|
|
18694
|
+
const rows = cache.query(appendLimit(config.sql, limit));
|
|
18695
|
+
deps.printData(config.mapRows?.(rows) ?? rows);
|
|
18696
|
+
} finally {
|
|
18697
|
+
cache.close();
|
|
17991
18698
|
}
|
|
17992
|
-
|
|
17993
|
-
|
|
17994
|
-
|
|
17995
|
-
|
|
17996
|
-
|
|
17997
|
-
|
|
17998
|
-
|
|
18699
|
+
}
|
|
18700
|
+
});
|
|
18701
|
+
};
|
|
18702
|
+
const buildMessages = (deps) => {
|
|
18703
|
+
return buildCacheRead(deps, {
|
|
18704
|
+
name: "messages",
|
|
18705
|
+
description: "List synced X DM messages [read] | usage: `social x messages --limit N`",
|
|
18706
|
+
collectionKey: "x_messages",
|
|
18707
|
+
limitMax: 100,
|
|
18708
|
+
operation: messagesOp,
|
|
18709
|
+
refreshEveryCall: true,
|
|
18710
|
+
sql: "SELECT m.*, p.handle AS sender_handle, p.display_name AS sender_name, p.avatar_url AS sender_avatar_url, p.headline AS sender_headline FROM x_messages m LEFT JOIN x_profiles p ON m.sender_id = p.provider_id ORDER BY m.created_at DESC",
|
|
18711
|
+
mapRows: withSender
|
|
18712
|
+
});
|
|
18713
|
+
};
|
|
18714
|
+
const buildFollowers = (deps) => buildCacheRead(deps, {
|
|
18715
|
+
name: "followers",
|
|
18716
|
+
description: "List synced X followers [read] | usage: `social x followers --limit N`",
|
|
18717
|
+
collectionKey: "x_followers",
|
|
18718
|
+
limitMax: 1e3,
|
|
18719
|
+
operation: operationForCommand("followers"),
|
|
18720
|
+
sql: "SELECT * FROM x_followers ORDER BY synced_at DESC"
|
|
18721
|
+
});
|
|
18722
|
+
const buildFollowing = (deps) => buildCacheRead(deps, {
|
|
18723
|
+
name: "following",
|
|
18724
|
+
description: "List synced X following [read] | usage: `social x following --limit N`",
|
|
18725
|
+
collectionKey: "x_following",
|
|
18726
|
+
limitMax: 1e3,
|
|
18727
|
+
operation: operationForCommand("following"),
|
|
18728
|
+
sql: "SELECT * FROM x_following ORDER BY synced_at DESC"
|
|
18729
|
+
});
|
|
18730
|
+
const buildBookmarks = (deps) => buildCacheRead(deps, {
|
|
18731
|
+
name: "bookmarks",
|
|
18732
|
+
description: "List synced X bookmarks [read] | usage: `social x bookmarks --limit N`",
|
|
18733
|
+
collectionKey: "x_bookmarks",
|
|
18734
|
+
limitMax: 100,
|
|
18735
|
+
operation: operationForCommand("bookmarks"),
|
|
18736
|
+
refreshEveryCall: true,
|
|
18737
|
+
sql: "SELECT * FROM x_bookmarks ORDER BY created_at DESC"
|
|
18738
|
+
});
|
|
18739
|
+
const hasTargetArg = (args) => {
|
|
18740
|
+
const positional = Array.isArray(args._) ? args._ : [];
|
|
18741
|
+
return args.target !== void 0 || positional.length > 0;
|
|
18742
|
+
};
|
|
18743
|
+
const buildTweets = (deps) => {
|
|
18744
|
+
const operation = operationForCommand("tweets");
|
|
18745
|
+
const liveTweets = defineOperationCommand(operation, deps);
|
|
18746
|
+
const cacheTweets = buildCacheRead(deps, {
|
|
18747
|
+
name: "tweets",
|
|
18748
|
+
description: "List synced X tweets [read] | usage: `social x tweets --limit N`",
|
|
18749
|
+
collectionKey: "x_tweets",
|
|
18750
|
+
limitMax: 100,
|
|
18751
|
+
operation,
|
|
18752
|
+
refreshEveryCall: true,
|
|
18753
|
+
sql: "SELECT * FROM x_tweets ORDER BY created_at DESC"
|
|
18754
|
+
});
|
|
18755
|
+
const args = {
|
|
18756
|
+
...liveTweets.args ?? {},
|
|
18757
|
+
...cacheReadArgs()
|
|
18758
|
+
};
|
|
18759
|
+
return defineCommand({
|
|
18760
|
+
meta: liveTweets.meta,
|
|
18761
|
+
args,
|
|
18762
|
+
run: async (ctx) => {
|
|
18763
|
+
const runArgs = ctx.args;
|
|
18764
|
+
if (hasTargetArg(runArgs)) {
|
|
18765
|
+
await liveTweets.run?.(ctx);
|
|
18766
|
+
return;
|
|
18767
|
+
}
|
|
18768
|
+
rejectUnknownArgs(runArgs, args);
|
|
18769
|
+
await cacheTweets.run?.(ctx);
|
|
17999
18770
|
}
|
|
18000
18771
|
});
|
|
18001
18772
|
};
|
|
@@ -18189,6 +18960,10 @@ const buildReplies = (deps) => {
|
|
|
18189
18960
|
};
|
|
18190
18961
|
const createXCommands = (deps) => {
|
|
18191
18962
|
const messages = buildMessages(deps);
|
|
18963
|
+
const followers = buildFollowers(deps);
|
|
18964
|
+
const following = buildFollowing(deps);
|
|
18965
|
+
const bookmarks = buildBookmarks(deps);
|
|
18966
|
+
const tweets = buildTweets(deps);
|
|
18192
18967
|
const message = buildMessage(deps);
|
|
18193
18968
|
const replies = buildReplies(deps);
|
|
18194
18969
|
const sync = buildSync(deps, "x");
|
|
@@ -18200,30 +18975,50 @@ const createXCommands = (deps) => {
|
|
|
18200
18975
|
index
|
|
18201
18976
|
})).filter((entry) => !hiddenRootCommands.has(entry.name));
|
|
18202
18977
|
entries.push({
|
|
18978
|
+
name: "followers",
|
|
18979
|
+
command: followers,
|
|
18980
|
+
rank: 0,
|
|
18981
|
+
index: X_OPERATIONS.length
|
|
18982
|
+
}, {
|
|
18983
|
+
name: "following",
|
|
18984
|
+
command: following,
|
|
18985
|
+
rank: 0,
|
|
18986
|
+
index: X_OPERATIONS.length + 1
|
|
18987
|
+
}, {
|
|
18203
18988
|
name: "messages",
|
|
18204
18989
|
command: messages,
|
|
18205
18990
|
rank: 0,
|
|
18206
|
-
index: X_OPERATIONS.length
|
|
18991
|
+
index: X_OPERATIONS.length + 2
|
|
18992
|
+
}, {
|
|
18993
|
+
name: "bookmarks",
|
|
18994
|
+
command: bookmarks,
|
|
18995
|
+
rank: 0,
|
|
18996
|
+
index: X_OPERATIONS.length + 3
|
|
18997
|
+
}, {
|
|
18998
|
+
name: "tweets",
|
|
18999
|
+
command: tweets,
|
|
19000
|
+
rank: 0,
|
|
19001
|
+
index: X_OPERATIONS.length + 4
|
|
18207
19002
|
}, {
|
|
18208
19003
|
name: "message",
|
|
18209
19004
|
command: message,
|
|
18210
19005
|
rank: 1,
|
|
18211
|
-
index: X_OPERATIONS.length +
|
|
19006
|
+
index: X_OPERATIONS.length + 5
|
|
18212
19007
|
}, {
|
|
18213
19008
|
name: "replies",
|
|
18214
19009
|
command: replies,
|
|
18215
19010
|
rank: 0,
|
|
18216
|
-
index: X_OPERATIONS.length +
|
|
19011
|
+
index: X_OPERATIONS.length + 6
|
|
18217
19012
|
}, {
|
|
18218
19013
|
name: "sync",
|
|
18219
19014
|
command: sync,
|
|
18220
19015
|
rank: 0,
|
|
18221
|
-
index: X_OPERATIONS.length +
|
|
19016
|
+
index: X_OPERATIONS.length + 7
|
|
18222
19017
|
}, {
|
|
18223
19018
|
name: "sql",
|
|
18224
19019
|
command: sql,
|
|
18225
19020
|
rank: 0,
|
|
18226
|
-
index: X_OPERATIONS.length +
|
|
19021
|
+
index: X_OPERATIONS.length + 8
|
|
18227
19022
|
});
|
|
18228
19023
|
entries.sort((left, right) => left.rank - right.rank || left.index - right.index);
|
|
18229
19024
|
return defineCommand({
|
|
@@ -18412,19 +19207,6 @@ const disconnectLinkedinAccount = async (deps, identifier) => {
|
|
|
18412
19207
|
};
|
|
18413
19208
|
};
|
|
18414
19209
|
//#endregion
|
|
18415
|
-
//#region ../../packages/lib/src/url.ts
|
|
18416
|
-
const stripTrailingSlashes = (value) => value.replace(/\/+$/, "");
|
|
18417
|
-
const stripLeadingSlashes = (value) => value.replace(/^\/+/, "");
|
|
18418
|
-
const ensureTrailingSlash = (value) => value.endsWith("/") ? value : `${value}/`;
|
|
18419
|
-
const absoluteURLPathPattern = /^[a-z][a-z0-9+.-]*:/i;
|
|
18420
|
-
const protocolRelativeURLPathPattern = /^\/\//;
|
|
18421
|
-
const createURL = (baseURL, path) => {
|
|
18422
|
-
const trimmedPath = path.trimStart();
|
|
18423
|
-
if (absoluteURLPathPattern.test(trimmedPath) || protocolRelativeURLPathPattern.test(trimmedPath)) throw new Error("createURL: path must be relative, not absolute or protocol-relative");
|
|
18424
|
-
return new URL(stripLeadingSlashes(path), ensureTrailingSlash(stripTrailingSlashes(baseURL.toString())));
|
|
18425
|
-
};
|
|
18426
|
-
const createAbsoluteURL = (baseURL, path) => createURL(baseURL, path).toString();
|
|
18427
|
-
//#endregion
|
|
18428
19210
|
//#region ../../packages/x/src/cli/lifecycle.ts
|
|
18429
19211
|
const CONNECT_TIMEOUT_MS = 300 * 1e3;
|
|
18430
19212
|
const SEAT_POLL_TIMEOUT_MS$1 = 600 * 1e3;
|
|
@@ -21110,7 +21892,7 @@ const createInstance = (defaults) => {
|
|
|
21110
21892
|
const ky = createInstance();
|
|
21111
21893
|
//#endregion
|
|
21112
21894
|
//#region package.json
|
|
21113
|
-
var version$1 = "0.
|
|
21895
|
+
var version$1 = "0.6.0";
|
|
21114
21896
|
//#endregion
|
|
21115
21897
|
//#region src/lib/env.ts
|
|
21116
21898
|
const URLWithTrailingSlash = url().transform(ensureTrailingSlash);
|
|
@@ -21237,6 +22019,8 @@ const clearCredentials = async (apiURL = activeAPIURL()) => {
|
|
|
21237
22019
|
};
|
|
21238
22020
|
//#endregion
|
|
21239
22021
|
//#region src/lib/api.ts
|
|
22022
|
+
const LINKEDIN_RATE_LIMIT_RETRY_LIMIT = 25;
|
|
22023
|
+
const LINKEDIN_RATE_LIMIT_MAX_FALLBACK_SECONDS = 60;
|
|
21240
22024
|
var CLIAPIError = class extends Error {
|
|
21241
22025
|
status;
|
|
21242
22026
|
body;
|
|
@@ -21254,14 +22038,45 @@ const readBearerToken = async () => {
|
|
|
21254
22038
|
if (!credentials?.accessToken) throw new AuthError("Not logged in. Run `social account login` first.");
|
|
21255
22039
|
return credentials.accessToken;
|
|
21256
22040
|
};
|
|
21257
|
-
const
|
|
22041
|
+
const linkedInRateLimitFallbackDelaySeconds = (retryCount) => Math.min(2 ** Math.max(0, retryCount - 1), LINKEDIN_RATE_LIMIT_MAX_FALLBACK_SECONDS);
|
|
22042
|
+
const linkedInRateLimitRetryDelaySeconds = (retryCount, retryAfterSeconds) => retryAfterSeconds ?? linkedInRateLimitFallbackDelaySeconds(retryCount);
|
|
22043
|
+
const isLinkedInProxyRequest = (request) => new URL(request.url).pathname.includes("/linkedin/");
|
|
22044
|
+
const shouldRetryLinkedInRateLimit = (args) => args.error instanceof HTTPError && args.error.response.status === 429 && isLinkedInProxyRequest(args.error.request) && args.retryCount <= LINKEDIN_RATE_LIMIT_RETRY_LIMIT;
|
|
22045
|
+
const writeRateLimitRetryNotice = (args) => {
|
|
22046
|
+
process.stderr.write(`LinkedIn rate limited; waiting ${args.delaySeconds}s before retry ${args.retryCount}/${LINKEDIN_RATE_LIMIT_RETRY_LIMIT}.\n`);
|
|
22047
|
+
};
|
|
22048
|
+
const createAPI = (opts = {}) => ky.create({
|
|
21258
22049
|
prefix: env.SOCIAL_API_URL,
|
|
22050
|
+
fetch: opts.fetch,
|
|
21259
22051
|
timeout: false,
|
|
21260
|
-
retry:
|
|
22052
|
+
retry: {
|
|
22053
|
+
limit: LINKEDIN_RATE_LIMIT_RETRY_LIMIT,
|
|
22054
|
+
methods: [
|
|
22055
|
+
"get",
|
|
22056
|
+
"post",
|
|
22057
|
+
"patch",
|
|
22058
|
+
"put",
|
|
22059
|
+
"delete"
|
|
22060
|
+
],
|
|
22061
|
+
statusCodes: [429],
|
|
22062
|
+
afterStatusCodes: [429],
|
|
22063
|
+
delay: (retryCount) => linkedInRateLimitFallbackDelaySeconds(retryCount) * 1e3,
|
|
22064
|
+
shouldRetry: ({ error, retryCount }) => {
|
|
22065
|
+
if (!shouldRetryLinkedInRateLimit({
|
|
22066
|
+
error,
|
|
22067
|
+
retryCount
|
|
22068
|
+
})) return false;
|
|
22069
|
+
const retryAfterSeconds = error instanceof HTTPError ? retryAfterSecondsFrom(error.response.headers.get("retry-after")) : void 0;
|
|
22070
|
+
(opts.onRateLimitRetry ?? writeRateLimitRetryNotice)({
|
|
22071
|
+
delaySeconds: linkedInRateLimitRetryDelaySeconds(retryCount, retryAfterSeconds),
|
|
22072
|
+
retryCount
|
|
22073
|
+
});
|
|
22074
|
+
}
|
|
22075
|
+
},
|
|
21261
22076
|
hooks: {
|
|
21262
22077
|
beforeRequest: [async ({ request, options }) => {
|
|
21263
22078
|
if (typeof options.body === "string" && !request.headers.has("content-type")) request.headers.set("content-type", "application/json");
|
|
21264
|
-
request.headers.set("authorization", `Bearer ${await readBearerToken()}`);
|
|
22079
|
+
request.headers.set("authorization", `Bearer ${await (opts.readBearerToken ?? readBearerToken)()}`);
|
|
21265
22080
|
request.headers.set("x-social-surface", "cli");
|
|
21266
22081
|
}],
|
|
21267
22082
|
beforeError: [({ error }) => {
|
|
@@ -21271,6 +22086,7 @@ const api = ky.create({
|
|
|
21271
22086
|
}]
|
|
21272
22087
|
}
|
|
21273
22088
|
});
|
|
22089
|
+
const api = createAPI();
|
|
21274
22090
|
const authAPI = ky.create({
|
|
21275
22091
|
prefix: apiURL("auth"),
|
|
21276
22092
|
timeout: false,
|
|
@@ -21628,7 +22444,7 @@ const totalCountFrom = (contract, data) => {
|
|
|
21628
22444
|
if (!isRecord$3(pagination)) return;
|
|
21629
22445
|
return numberAtPath(data, pagination.totalCountPath);
|
|
21630
22446
|
};
|
|
21631
|
-
const resultEnvelope = ({ account, contract, data, resolved = [], responseHeaders }) => {
|
|
22447
|
+
const resultEnvelope = ({ account, contract, data, resolved = [], responseHeaders, usage }) => {
|
|
21632
22448
|
const payload = outputPayloadFor(data, itemURLTemplateFrom(contract), idFieldFrom(contract));
|
|
21633
22449
|
const cache = cacheFromHeaders(responseHeaders);
|
|
21634
22450
|
const itemCount = "items" in payload && Array.isArray(payload.items) ? payload.items.length : 1;
|
|
@@ -21645,7 +22461,8 @@ const resultEnvelope = ({ account, contract, data, resolved = [], responseHeader
|
|
|
21645
22461
|
cost: costFromContract(contract, itemCount, cache.hit, responseHeaders),
|
|
21646
22462
|
cache,
|
|
21647
22463
|
...cursor === void 0 ? {} : { cursor },
|
|
21648
|
-
...totalCount === void 0 ? {} : { totalCount }
|
|
22464
|
+
...totalCount === void 0 ? {} : { totalCount },
|
|
22465
|
+
...usage === void 0 ? {} : { usage }
|
|
21649
22466
|
}
|
|
21650
22467
|
};
|
|
21651
22468
|
};
|
|
@@ -22057,13 +22874,8 @@ const BillingOutput = object({
|
|
|
22057
22874
|
usage: object({
|
|
22058
22875
|
calls: object({ total: number().int().nonnegative() }),
|
|
22059
22876
|
credits: object({
|
|
22060
|
-
|
|
22061
|
-
used: number().nonnegative()
|
|
22062
|
-
additional: number().nonnegative()
|
|
22063
|
-
}),
|
|
22064
|
-
billing: object({
|
|
22065
|
-
seats: number().nonnegative(),
|
|
22066
|
-
usage: number().nonnegative()
|
|
22877
|
+
included: number().nonnegative(),
|
|
22878
|
+
used: number().nonnegative()
|
|
22067
22879
|
})
|
|
22068
22880
|
})
|
|
22069
22881
|
});
|
|
@@ -24033,7 +24845,7 @@ async function addSourceContext(frames) {
|
|
|
24033
24845
|
function getContextLinesFromFile(path, ranges, output) {
|
|
24034
24846
|
return new Promise((resolve) => {
|
|
24035
24847
|
const stream = createReadStream(path);
|
|
24036
|
-
const lineReaded = createInterface
|
|
24848
|
+
const lineReaded = createInterface({ input: stream });
|
|
24037
24849
|
function destroyStreamAndResolve() {
|
|
24038
24850
|
stream.destroy();
|
|
24039
24851
|
resolve();
|
|
@@ -28234,13 +29046,8 @@ const UsageLogsOutput = object({
|
|
|
28234
29046
|
const UsageSummaryOutput = object({
|
|
28235
29047
|
calls: object({ total: number().int().nonnegative() }),
|
|
28236
29048
|
credits: object({
|
|
28237
|
-
|
|
28238
|
-
used: number().nonnegative()
|
|
28239
|
-
additional: number().nonnegative()
|
|
28240
|
-
}),
|
|
28241
|
-
billing: object({
|
|
28242
|
-
seats: number().nonnegative(),
|
|
28243
|
-
usage: number().nonnegative()
|
|
29049
|
+
included: number().nonnegative(),
|
|
29050
|
+
used: number().nonnegative()
|
|
28244
29051
|
})
|
|
28245
29052
|
});
|
|
28246
29053
|
const createUsageLogsCommand = (deps = {}) => defineCommand({
|
|
@@ -28733,6 +29540,17 @@ const commandHeadersFrom = (defaults, args = {}) => {
|
|
|
28733
29540
|
};
|
|
28734
29541
|
const createCLIDeps = () => {
|
|
28735
29542
|
const client = createORPCAPIClient();
|
|
29543
|
+
const printResultWithUsage = async (args) => {
|
|
29544
|
+
try {
|
|
29545
|
+
const usage = await client.usage.summary({});
|
|
29546
|
+
printResult({
|
|
29547
|
+
...args,
|
|
29548
|
+
usage
|
|
29549
|
+
});
|
|
29550
|
+
} catch {
|
|
29551
|
+
printResult(args);
|
|
29552
|
+
}
|
|
29553
|
+
};
|
|
28736
29554
|
const resolveAccount = async ({ account, platform }) => {
|
|
28737
29555
|
if (account) resolveAccountId({ account });
|
|
28738
29556
|
const accounts = await client.accounts.list({
|
|
@@ -28841,7 +29659,7 @@ const createCLIDeps = () => {
|
|
|
28841
29659
|
}),
|
|
28842
29660
|
commandHeaders: async (args) => commandHeadersFrom({ "Cache-Control": await cacheControlHeaderFromConfig() }, args),
|
|
28843
29661
|
printData,
|
|
28844
|
-
printResult,
|
|
29662
|
+
printResult: printResultWithUsage,
|
|
28845
29663
|
resolveAccount,
|
|
28846
29664
|
resolveAccountId: async (args) => resolveAccountId(args),
|
|
28847
29665
|
resolveOwnXAccount: async ({ account }) => {
|