@usesocial/cli 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/README.md +62 -36
- package/dist/index.mjs +1415 -1012
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/dist/index.mjs
CHANGED
|
@@ -129,805 +129,1014 @@ var CancelledError = class extends Error {
|
|
|
129
129
|
}
|
|
130
130
|
};
|
|
131
131
|
//#endregion
|
|
132
|
-
//#region ../../packages/sync-engine/src/
|
|
133
|
-
const
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
"sql": "CREATE TABLE `li_profiles` (\n `provider_id` text PRIMARY KEY NOT NULL,\n `handle` text,\n `display_name` text NOT NULL,\n `avatar_url` text,\n `headline` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `li_profiles_handle_idx` ON `li_profiles` (`handle`);--> statement-breakpoint\nCREATE TABLE `x_messages` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `dm_conversation_id` text,\n `event_type` text,\n `sender_id` text,\n `text` text,\n `participant_ids` text,\n `attachments` text,\n `entities` text,\n `referenced_tweets` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `x_messages_created_at_idx` ON `x_messages` (`created_at`);--> statement-breakpoint\nCREATE INDEX `x_messages_dm_conversation_id_idx` ON `x_messages` (`dm_conversation_id`);--> statement-breakpoint\nCREATE TABLE `x_profiles` (\n `provider_id` text PRIMARY KEY NOT NULL,\n `handle` text,\n `display_name` text NOT NULL,\n `avatar_url` text,\n `headline` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `x_profiles_handle_idx` ON `x_profiles` (`handle`);"
|
|
141
|
-
},
|
|
142
|
-
{
|
|
143
|
-
"tag": "0002_previous_spitfire",
|
|
144
|
-
"sql": "ALTER TABLE `li_connections` ADD `user_headline` text;--> statement-breakpoint\nALTER TABLE `li_requests` ADD `user_headline` text;--> statement-breakpoint\nALTER TABLE `x_bookmarks` DROP COLUMN `edit_controls_editable_until`;--> statement-breakpoint\nALTER TABLE `x_bookmarks` DROP COLUMN `edit_controls_edits_remaining`;--> statement-breakpoint\nALTER TABLE `x_bookmarks` DROP COLUMN `edit_controls_is_edit_eligible`;--> statement-breakpoint\nALTER TABLE `x_tweets` DROP COLUMN `edit_controls_editable_until`;--> statement-breakpoint\nALTER TABLE `x_tweets` DROP COLUMN `edit_controls_edits_remaining`;--> statement-breakpoint\nALTER TABLE `x_tweets` DROP COLUMN `edit_controls_is_edit_eligible`;"
|
|
145
|
-
},
|
|
146
|
-
{
|
|
147
|
-
"tag": "0003_wonderful_makkari",
|
|
148
|
-
"sql": "ALTER TABLE `li_conversations` RENAME COLUMN \"headline\" TO \"user_headline\";--> statement-breakpoint\nCREATE INDEX `li_conversations_user_id_last_message_timestamp_idx` ON `li_conversations` (`user_id`,`last_message_timestamp`);--> statement-breakpoint\nCREATE INDEX `li_conversations_user_public_identifier_last_message_timestamp_idx` ON `li_conversations` (`user_public_identifier`,`last_message_timestamp`);--> statement-breakpoint\nCREATE INDEX `li_conversations_user_display_name_idx` ON `li_conversations` (`user_display_name`);--> statement-breakpoint\nCREATE INDEX `li_conversations_is_archived_last_message_timestamp_idx` ON `li_conversations` (`is_archived`,`last_message_timestamp`);--> statement-breakpoint\nDROP INDEX `li_messages_chat_id_idx`;--> statement-breakpoint\nCREATE INDEX `li_messages_chat_id_timestamp_idx` ON `li_messages` (`chat_id`,`timestamp`);--> statement-breakpoint\nCREATE INDEX `li_messages_sender_id_timestamp_idx` ON `li_messages` (`sender_id`,`timestamp`);--> statement-breakpoint\nDROP INDEX `x_messages_dm_conversation_id_idx`;--> statement-breakpoint\nCREATE INDEX `x_messages_dm_conversation_id_created_at_idx` ON `x_messages` (`dm_conversation_id`,`created_at`);--> statement-breakpoint\nCREATE INDEX `x_messages_sender_id_created_at_idx` ON `x_messages` (`sender_id`,`created_at`);--> statement-breakpoint\nCREATE INDEX `li_connections_synced_at_idx` ON `li_connections` (`synced_at`);--> statement-breakpoint\nCREATE INDEX `li_connections_user_id_idx` ON `li_connections` (`user_id`);--> statement-breakpoint\nCREATE INDEX `li_connections_user_public_identifier_idx` ON `li_connections` (`user_public_identifier`);--> statement-breakpoint\nCREATE INDEX `li_connections_user_display_name_idx` ON `li_connections` (`user_display_name`);--> statement-breakpoint\nCREATE INDEX `li_posts_author_id_created_at_idx` ON `li_posts` (`author_id`,`created_at`);--> statement-breakpoint\nCREATE INDEX `li_posts_author_public_identifier_created_at_idx` ON `li_posts` (`author_public_identifier`,`created_at`);--> statement-breakpoint\nCREATE INDEX `li_requests_type_synced_at_idx` ON `li_requests` (`type`,`synced_at`);--> statement-breakpoint\nCREATE INDEX `li_requests_user_id_idx` ON `li_requests` (`user_id`);--> statement-breakpoint\nCREATE INDEX `li_requests_user_public_identifier_idx` ON `li_requests` (`user_public_identifier`);--> statement-breakpoint\nCREATE INDEX `li_requests_user_display_name_idx` ON `li_requests` (`user_display_name`);--> statement-breakpoint\nCREATE INDEX `x_bookmarks_author_id_created_at_idx` ON `x_bookmarks` (`author_id`,`created_at`);--> statement-breakpoint\nCREATE INDEX `x_bookmarks_conversation_id_created_at_idx` ON `x_bookmarks` (`conversation_id`,`created_at`);--> statement-breakpoint\nCREATE INDEX `x_followers_synced_at_idx` ON `x_followers` (`synced_at`);--> statement-breakpoint\nCREATE INDEX `x_followers_username_idx` ON `x_followers` (`username`);--> statement-breakpoint\nCREATE INDEX `x_followers_verified_idx` ON `x_followers` (`verified`);--> statement-breakpoint\nCREATE INDEX `x_followers_verified_type_idx` ON `x_followers` (`verified_type`);--> statement-breakpoint\nCREATE INDEX `x_following_synced_at_idx` ON `x_following` (`synced_at`);--> statement-breakpoint\nCREATE INDEX `x_following_username_idx` ON `x_following` (`username`);--> statement-breakpoint\nCREATE INDEX `x_following_verified_idx` ON `x_following` (`verified`);--> statement-breakpoint\nCREATE INDEX `x_following_verified_type_idx` ON `x_following` (`verified_type`);--> statement-breakpoint\nCREATE INDEX `x_tweets_author_id_created_at_idx` ON `x_tweets` (`author_id`,`created_at`);--> statement-breakpoint\nCREATE INDEX `x_tweets_conversation_id_created_at_idx` ON `x_tweets` (`conversation_id`,`created_at`);"
|
|
149
|
-
},
|
|
150
|
-
{
|
|
151
|
-
"tag": "0004_quiet_prism",
|
|
152
|
-
"sql": "ALTER TABLE `x_messages` RENAME TO `x_raw_messages`;--> statement-breakpoint\nCREATE VIEW `x_messages` AS\nSELECT m.*, p.handle AS sender_handle, p.display_name AS sender_name,\n p.avatar_url AS sender_avatar_url, p.headline AS sender_headline\nFROM `x_raw_messages` m LEFT JOIN `x_profiles` p ON m.sender_id = p.provider_id;\n"
|
|
153
|
-
}
|
|
154
|
-
];
|
|
155
|
-
//#endregion
|
|
156
|
-
//#region ../../packages/sync-engine/src/migrate.ts
|
|
157
|
-
const readUserVersion = (db) => {
|
|
158
|
-
const version = db.prepare("PRAGMA user_version").get()?.user_version;
|
|
159
|
-
return typeof version === "number" ? version : Number(version ?? 0);
|
|
160
|
-
};
|
|
161
|
-
const setUserVersion = (db, version) => {
|
|
162
|
-
db.exec(`PRAGMA user_version = ${version}`);
|
|
163
|
-
};
|
|
164
|
-
const migrate = (db) => {
|
|
165
|
-
const currentVersion = readUserVersion(db);
|
|
166
|
-
if (currentVersion >= MIGRATIONS.length) return {
|
|
167
|
-
fromVersion: currentVersion,
|
|
168
|
-
toVersion: currentVersion,
|
|
169
|
-
applied: 0
|
|
170
|
-
};
|
|
171
|
-
for (let index = currentVersion; index < MIGRATIONS.length; index += 1) {
|
|
172
|
-
const migration = MIGRATIONS[index];
|
|
173
|
-
if (!migration) throw new Error(`Missing migration at index ${index}`);
|
|
174
|
-
db.transaction(() => {
|
|
175
|
-
db.exec(migration.sql);
|
|
176
|
-
setUserVersion(db, index + 1);
|
|
177
|
-
});
|
|
178
|
-
}
|
|
179
|
-
return {
|
|
180
|
-
fromVersion: currentVersion,
|
|
181
|
-
toVersion: MIGRATIONS.length,
|
|
182
|
-
applied: MIGRATIONS.length - currentVersion
|
|
183
|
-
};
|
|
184
|
-
};
|
|
185
|
-
const assertAtLatest = (db) => {
|
|
186
|
-
const currentVersion = readUserVersion(db);
|
|
187
|
-
if (currentVersion < MIGRATIONS.length) throw new Error(`Local SQLite cache schema is at version ${currentVersion}; expected ${MIGRATIONS.length}`);
|
|
132
|
+
//#region ../../packages/sync-engine/src/collections/linkedin.ts
|
|
133
|
+
const linkedinCursorPagination = {
|
|
134
|
+
style: "cursor",
|
|
135
|
+
limitParam: "limit",
|
|
136
|
+
cursorParam: "cursor",
|
|
137
|
+
cursorPath: "next_cursor",
|
|
138
|
+
totalCountPath: "total_count",
|
|
139
|
+
itemsPath: "data"
|
|
188
140
|
};
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
if (typeof value === "boolean") return value ? 1 : 0;
|
|
196
|
-
return value;
|
|
141
|
+
const linkedinRelationsCursorPagination = {
|
|
142
|
+
style: "cursor",
|
|
143
|
+
limitParam: "limit",
|
|
144
|
+
cursorParam: "cursor",
|
|
145
|
+
cursorPath: "next_cursor",
|
|
146
|
+
itemsPath: "data"
|
|
197
147
|
};
|
|
198
|
-
const
|
|
199
|
-
|
|
200
|
-
|
|
148
|
+
const asRecord$1 = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
149
|
+
const read$1 = (value) => value === null || value === void 0 ? null : value;
|
|
150
|
+
const readReaction = (value) => typeof value === "string" ? value : null;
|
|
151
|
+
const trimmed$1 = (value) => typeof value === "string" ? value.trim() || void 0 : void 0;
|
|
152
|
+
const userHeadline = (user) => {
|
|
153
|
+
const specifics = asRecord$1(user.specifics);
|
|
154
|
+
return trimmed$1(user.description) ?? trimmed$1(user.headline) ?? trimmed$1(specifics.description) ?? trimmed$1(specifics.headline) ?? trimmed$1(specifics.occupation) ?? null;
|
|
201
155
|
};
|
|
202
|
-
const
|
|
203
|
-
|
|
204
|
-
|
|
156
|
+
const isoToEpochMs$1 = (value) => {
|
|
157
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
158
|
+
if (typeof value !== "string") return null;
|
|
159
|
+
const epochMs = Date.parse(value);
|
|
160
|
+
return Number.isNaN(epochMs) ? null : epochMs;
|
|
205
161
|
};
|
|
206
|
-
const
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
all: (params) => invokeWithParams(statement.all.bind(statement), params)
|
|
210
|
-
});
|
|
211
|
-
const openRuntimeDatabase = (path, opts) => {
|
|
212
|
-
if (isBunRuntime()) {
|
|
213
|
-
const { Database } = loadModule("bun:sqlite");
|
|
214
|
-
return new Database(path, {
|
|
215
|
-
readonly: opts?.readonly,
|
|
216
|
-
strict: true
|
|
217
|
-
});
|
|
218
|
-
}
|
|
219
|
-
const { DatabaseSync } = loadModule("node:sqlite");
|
|
220
|
-
return opts?.readonly ? new DatabaseSync(path, { readOnly: true }) : new DatabaseSync(path);
|
|
162
|
+
const toBool$1 = (value) => {
|
|
163
|
+
if (typeof value !== "boolean") return null;
|
|
164
|
+
return value ? 1 : 0;
|
|
221
165
|
};
|
|
222
|
-
const
|
|
223
|
-
const
|
|
224
|
-
const exec = (sql) => {
|
|
225
|
-
database.exec(sql);
|
|
226
|
-
};
|
|
166
|
+
const liftConnection = (item) => {
|
|
167
|
+
const user = asRecord$1(item.user);
|
|
227
168
|
return {
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
234
|
-
|
|
235
|
-
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
}
|
|
169
|
+
id: read$1(item.id),
|
|
170
|
+
created_at: isoToEpochMs$1(item.created_at),
|
|
171
|
+
object: read$1(item.object),
|
|
172
|
+
url: read$1(item.url),
|
|
173
|
+
user_id: read$1(user.id),
|
|
174
|
+
user_object: read$1(user.object),
|
|
175
|
+
user_type: read$1(user.type),
|
|
176
|
+
user_display_name: read$1(user.display_name),
|
|
177
|
+
user_public_identifier: read$1(user.public_identifier),
|
|
178
|
+
user_profile_url: read$1(user.profile_url),
|
|
179
|
+
user_public_picture_url: read$1(user.public_picture_url),
|
|
180
|
+
user_description: read$1(user.description),
|
|
181
|
+
user_headline: userHeadline(user),
|
|
182
|
+
raw: JSON.stringify(item)
|
|
243
183
|
};
|
|
244
184
|
};
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
|
|
260
|
-
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
185
|
+
const liftConversation = (item) => {
|
|
186
|
+
const user = asRecord$1(item.user);
|
|
187
|
+
const userSpecifics = asRecord$1(user.specifics);
|
|
188
|
+
return {
|
|
189
|
+
id: read$1(item.id),
|
|
190
|
+
created_at: isoToEpochMs$1(item.created_at),
|
|
191
|
+
last_message_timestamp: isoToEpochMs$1(item.last_message_timestamp),
|
|
192
|
+
muted_until: isoToEpochMs$1(item.muted_until),
|
|
193
|
+
creator_id: read$1(item.creator_id),
|
|
194
|
+
user_headline: userHeadline(user),
|
|
195
|
+
image_url: read$1(item.image_url),
|
|
196
|
+
name: read$1(item.name),
|
|
197
|
+
object: read$1(item.object),
|
|
198
|
+
provider: read$1(item.provider),
|
|
199
|
+
type: read$1(item.type),
|
|
200
|
+
user_id: read$1(item.user_id ?? user.id),
|
|
201
|
+
is_1to1: toBool$1(item.is_1to1),
|
|
202
|
+
is_archived: toBool$1(item.is_archived),
|
|
203
|
+
is_channel: toBool$1(item.is_channel),
|
|
204
|
+
is_group: toBool$1(item.is_group),
|
|
205
|
+
is_pinned: toBool$1(item.is_pinned),
|
|
206
|
+
is_readonly: toBool$1(item.is_readonly),
|
|
207
|
+
unread_count: read$1(item.unread_count),
|
|
208
|
+
user_object: read$1(user.object),
|
|
209
|
+
user_type: read$1(user.type),
|
|
210
|
+
user_display_name: read$1(user.display_name),
|
|
211
|
+
user_public_identifier: read$1(user.public_identifier),
|
|
212
|
+
user_profile_url: read$1(user.profile_url),
|
|
213
|
+
user_public_picture_url: read$1(user.public_picture_url),
|
|
214
|
+
user_description: read$1(user.description),
|
|
215
|
+
user_specifics_network_distance: read$1(userSpecifics.network_distance),
|
|
216
|
+
raw: JSON.stringify(item)
|
|
217
|
+
};
|
|
276
218
|
};
|
|
277
|
-
const
|
|
278
|
-
const
|
|
279
|
-
const
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
283
|
-
|
|
219
|
+
const liftMessage = (item) => {
|
|
220
|
+
const sender = asRecord$1(item.sender);
|
|
221
|
+
const senderSpecifics = asRecord$1(sender.specifics);
|
|
222
|
+
return {
|
|
223
|
+
id: read$1(item.id),
|
|
224
|
+
timestamp: isoToEpochMs$1(item.timestamp),
|
|
225
|
+
chat_id: read$1(item.chat_id ?? null),
|
|
226
|
+
object: read$1(item.object),
|
|
227
|
+
provider: read$1(item.provider),
|
|
228
|
+
text: read$1(item.text),
|
|
229
|
+
sender_id: read$1(item.sender_id),
|
|
230
|
+
is_deleted: toBool$1(item.is_deleted),
|
|
231
|
+
is_delivered: toBool$1(item.is_delivered),
|
|
232
|
+
is_edited: toBool$1(item.is_edited),
|
|
233
|
+
is_event: toBool$1(item.is_event),
|
|
234
|
+
is_hidden: toBool$1(item.is_hidden),
|
|
235
|
+
is_mentionned: toBool$1(item.is_mentionned),
|
|
236
|
+
is_pinned: toBool$1(item.is_pinned),
|
|
237
|
+
is_seen: toBool$1(item.is_seen),
|
|
238
|
+
is_sender: toBool$1(item.is_sender),
|
|
239
|
+
sender_object: read$1(sender.object),
|
|
240
|
+
sender_type: read$1(sender.type),
|
|
241
|
+
sender_display_name: read$1(sender.display_name),
|
|
242
|
+
sender_public_identifier: read$1(sender.public_identifier),
|
|
243
|
+
sender_profile_url: read$1(sender.profile_url),
|
|
244
|
+
sender_public_picture_url: read$1(sender.public_picture_url),
|
|
245
|
+
sender_description: read$1(sender.description),
|
|
246
|
+
sender_specifics_network_distance: read$1(senderSpecifics.network_distance),
|
|
247
|
+
raw: JSON.stringify(item)
|
|
248
|
+
};
|
|
284
249
|
};
|
|
285
|
-
const
|
|
286
|
-
const
|
|
287
|
-
const
|
|
288
|
-
const
|
|
289
|
-
const
|
|
290
|
-
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
|
|
250
|
+
const liftPost = (item) => {
|
|
251
|
+
const analytics = asRecord$1(item.analytics);
|
|
252
|
+
const author = asRecord$1(item.author);
|
|
253
|
+
const authorSpecifics = asRecord$1(author.specifics);
|
|
254
|
+
const permissions = asRecord$1(item.permissions);
|
|
255
|
+
const poll = asRecord$1(item.poll);
|
|
256
|
+
const quotedPost = asRecord$1(item.quoted_post);
|
|
257
|
+
const quotedPostPermissions = asRecord$1(quotedPost.permissions);
|
|
258
|
+
const quotedPostAuthor = asRecord$1(quotedPost.author);
|
|
259
|
+
const quotedPostAuthorSpecifics = asRecord$1(quotedPostAuthor.specifics);
|
|
260
|
+
const repostedBy = asRecord$1(item.reposted_by);
|
|
261
|
+
return {
|
|
262
|
+
id: read$1(item.id),
|
|
263
|
+
created_at: isoToEpochMs$1(item.created_at),
|
|
264
|
+
object: read$1(item.object),
|
|
265
|
+
share_url: read$1(item.share_url),
|
|
266
|
+
text: read$1(item.text),
|
|
267
|
+
url: read$1(item.url),
|
|
268
|
+
is_repost: toBool$1(item.is_repost),
|
|
269
|
+
user_reacted: readReaction(item.user_reacted),
|
|
270
|
+
comments_counter: read$1(item.comments_counter),
|
|
271
|
+
reposts_counter: read$1(item.reposts_counter),
|
|
272
|
+
analytics_impressions_counter: read$1(analytics.impressions_counter),
|
|
273
|
+
author_id: read$1(author.id),
|
|
274
|
+
author_object: read$1(author.object),
|
|
275
|
+
author_type: read$1(author.type),
|
|
276
|
+
author_display_name: read$1(author.display_name),
|
|
277
|
+
author_public_identifier: read$1(author.public_identifier),
|
|
278
|
+
author_profile_url: read$1(author.profile_url),
|
|
279
|
+
author_public_picture_url: read$1(author.public_picture_url),
|
|
280
|
+
author_description: read$1(author.description),
|
|
281
|
+
author_specifics_network_distance: read$1(authorSpecifics.network_distance),
|
|
282
|
+
permissions_can_post_comments: toBool$1(permissions.can_post_comments),
|
|
283
|
+
permissions_can_react: toBool$1(permissions.can_react),
|
|
284
|
+
permissions_can_share: toBool$1(permissions.can_share),
|
|
285
|
+
poll_id: read$1(poll.id),
|
|
286
|
+
poll_question: read$1(poll.question),
|
|
287
|
+
poll_is_open: toBool$1(poll.is_open),
|
|
288
|
+
poll_total_votes_count: read$1(poll.total_votes_count),
|
|
289
|
+
quoted_post_id: read$1(quotedPost.id),
|
|
290
|
+
quoted_post_object: read$1(quotedPost.object),
|
|
291
|
+
quoted_post_created_at: isoToEpochMs$1(quotedPost.created_at),
|
|
292
|
+
quoted_post_text: read$1(quotedPost.text),
|
|
293
|
+
quoted_post_share_url: read$1(quotedPost.share_url),
|
|
294
|
+
quoted_post_user_reacted: readReaction(quotedPost.user_reacted),
|
|
295
|
+
quoted_post_comments_counter: read$1(quotedPost.comments_counter),
|
|
296
|
+
quoted_post_reposts_counter: read$1(quotedPost.reposts_counter),
|
|
297
|
+
quoted_post_permissions_can_post_comments: toBool$1(quotedPostPermissions.can_post_comments),
|
|
298
|
+
quoted_post_permissions_can_react: toBool$1(quotedPostPermissions.can_react),
|
|
299
|
+
quoted_post_permissions_can_share: toBool$1(quotedPostPermissions.can_share),
|
|
300
|
+
quoted_post_author_id: read$1(quotedPostAuthor.id),
|
|
301
|
+
quoted_post_author_object: read$1(quotedPostAuthor.object),
|
|
302
|
+
quoted_post_author_type: read$1(quotedPostAuthor.type),
|
|
303
|
+
quoted_post_author_display_name: read$1(quotedPostAuthor.display_name),
|
|
304
|
+
quoted_post_author_public_identifier: read$1(quotedPostAuthor.public_identifier),
|
|
305
|
+
quoted_post_author_profile_url: read$1(quotedPostAuthor.profile_url),
|
|
306
|
+
quoted_post_author_public_picture_url: read$1(quotedPostAuthor.public_picture_url),
|
|
307
|
+
quoted_post_author_description: read$1(quotedPostAuthor.description),
|
|
308
|
+
quoted_post_author_specifics_network_distance: read$1(quotedPostAuthorSpecifics.network_distance),
|
|
309
|
+
reposted_by_id: read$1(repostedBy.id),
|
|
310
|
+
reposted_by_object: read$1(repostedBy.object),
|
|
311
|
+
reposted_by_type: read$1(repostedBy.type),
|
|
312
|
+
reposted_by_display_name: read$1(repostedBy.display_name),
|
|
313
|
+
reposted_by_public_identifier: read$1(repostedBy.public_identifier),
|
|
314
|
+
reposted_by_profile_url: read$1(repostedBy.profile_url),
|
|
315
|
+
reposted_by_public_picture_url: read$1(repostedBy.public_picture_url),
|
|
316
|
+
raw: JSON.stringify(item)
|
|
317
|
+
};
|
|
318
|
+
};
|
|
319
|
+
const liftRequest = (item) => {
|
|
320
|
+
const user = asRecord$1(item.user);
|
|
321
|
+
return {
|
|
322
|
+
id: read$1(item.id),
|
|
323
|
+
created_at: isoToEpochMs$1(item.created_at),
|
|
324
|
+
object: read$1(item.object),
|
|
325
|
+
type: read$1(item.type ?? null),
|
|
326
|
+
user_id: read$1(user.id),
|
|
327
|
+
user_object: read$1(user.object),
|
|
328
|
+
user_type: read$1(user.type),
|
|
329
|
+
user_display_name: read$1(user.display_name),
|
|
330
|
+
user_public_identifier: read$1(user.public_identifier),
|
|
331
|
+
user_profile_url: read$1(user.profile_url),
|
|
332
|
+
user_public_picture_url: read$1(user.public_picture_url),
|
|
333
|
+
user_description: read$1(user.description),
|
|
334
|
+
user_headline: userHeadline(user),
|
|
335
|
+
raw: JSON.stringify(item)
|
|
336
|
+
};
|
|
337
|
+
};
|
|
338
|
+
const LINKEDIN_COLLECTIONS = [
|
|
339
|
+
{
|
|
340
|
+
key: "li_conversations",
|
|
341
|
+
table: "li_conversations_raw",
|
|
342
|
+
platform: "linkedin",
|
|
343
|
+
requests: [{ path: "/chats" }],
|
|
344
|
+
pagination: linkedinCursorPagination,
|
|
345
|
+
pageSize: 20,
|
|
346
|
+
sinceField: "last_message_timestamp",
|
|
347
|
+
lift: liftConversation
|
|
348
|
+
},
|
|
349
|
+
{
|
|
350
|
+
key: "li_messages",
|
|
351
|
+
table: "li_messages_raw",
|
|
352
|
+
platform: "linkedin",
|
|
353
|
+
requests: [{ path: "/chats/:chat_id/messages" }],
|
|
354
|
+
parentCollection: "li_conversations",
|
|
355
|
+
parentIdToken: ":chat_id",
|
|
356
|
+
pagination: linkedinCursorPagination,
|
|
357
|
+
pageSize: 250,
|
|
358
|
+
sinceField: "timestamp",
|
|
359
|
+
lift: liftMessage
|
|
360
|
+
},
|
|
361
|
+
{
|
|
362
|
+
key: "li_connections",
|
|
363
|
+
table: "li_connections_raw",
|
|
364
|
+
platform: "linkedin",
|
|
365
|
+
requests: [{ path: "/users/connections" }],
|
|
366
|
+
requiresFullInitialSync: true,
|
|
367
|
+
pagination: linkedinRelationsCursorPagination,
|
|
368
|
+
pageSize: 100,
|
|
369
|
+
sinceField: "created_at",
|
|
370
|
+
lift: liftConnection
|
|
371
|
+
},
|
|
372
|
+
{
|
|
373
|
+
key: "li_posts",
|
|
374
|
+
table: "li_posts_raw",
|
|
375
|
+
platform: "linkedin",
|
|
376
|
+
requests: [{ path: "/users/:identifier/posts" }],
|
|
377
|
+
ownIdToken: ":identifier",
|
|
378
|
+
pagination: linkedinCursorPagination,
|
|
379
|
+
pageSize: 50,
|
|
380
|
+
sinceField: "created_at",
|
|
381
|
+
lift: liftPost
|
|
382
|
+
},
|
|
383
|
+
{
|
|
384
|
+
key: "li_requests",
|
|
385
|
+
table: "li_requests_raw",
|
|
386
|
+
platform: "linkedin",
|
|
387
|
+
requests: [{
|
|
388
|
+
path: "/users/me/relation-requests/received",
|
|
389
|
+
bakedQuery: { type: "received" }
|
|
390
|
+
}, {
|
|
391
|
+
path: "/users/me/relation-requests/sent",
|
|
392
|
+
bakedQuery: { type: "sent" }
|
|
393
|
+
}],
|
|
394
|
+
pagination: linkedinRelationsCursorPagination,
|
|
395
|
+
pageSize: 100,
|
|
396
|
+
lift: liftRequest
|
|
397
|
+
}
|
|
398
|
+
];
|
|
399
|
+
const userTypeValues = ["individual", "organization"];
|
|
400
|
+
const LINKEDIN_CACHE_METADATA = {
|
|
401
|
+
joins: [
|
|
402
|
+
"li_*.user_id = li_profiles.provider_id",
|
|
403
|
+
"li_messages.conversation_id = li_conversations.id",
|
|
404
|
+
"li_messages.sender_id = li_profiles.provider_id",
|
|
405
|
+
"li_posts.author_id = li_profiles.provider_id"
|
|
406
|
+
],
|
|
407
|
+
enums: {
|
|
408
|
+
"li_connections.user_type": userTypeValues,
|
|
409
|
+
"li_conversations.type": ["1to1", "group"],
|
|
410
|
+
"li_conversations.user_type": userTypeValues,
|
|
411
|
+
"li_messages.sender_type": userTypeValues,
|
|
412
|
+
"li_posts.author_type": userTypeValues,
|
|
413
|
+
"li_requests.type": ["received", "sent"],
|
|
414
|
+
"li_requests.user_type": userTypeValues
|
|
415
|
+
},
|
|
416
|
+
indexed: {
|
|
417
|
+
li_connections: [
|
|
418
|
+
"created_at",
|
|
419
|
+
"user_id",
|
|
420
|
+
"user_username",
|
|
421
|
+
"user_name"
|
|
422
|
+
],
|
|
423
|
+
li_conversations: [
|
|
424
|
+
"created_at",
|
|
425
|
+
"last_message_at",
|
|
426
|
+
"user_id",
|
|
427
|
+
"user_username",
|
|
428
|
+
"user_name",
|
|
429
|
+
"is_archived"
|
|
430
|
+
],
|
|
431
|
+
li_messages: [
|
|
432
|
+
"created_at",
|
|
433
|
+
"conversation_id",
|
|
434
|
+
"sender_id"
|
|
435
|
+
],
|
|
436
|
+
li_posts: [
|
|
437
|
+
"created_at",
|
|
438
|
+
"author_id",
|
|
439
|
+
"author_username"
|
|
440
|
+
],
|
|
441
|
+
li_profiles: ["username"],
|
|
442
|
+
li_requests: [
|
|
443
|
+
"created_at",
|
|
444
|
+
"type",
|
|
445
|
+
"user_id",
|
|
446
|
+
"user_username",
|
|
447
|
+
"user_name"
|
|
448
|
+
]
|
|
449
|
+
}
|
|
450
|
+
};
|
|
451
|
+
//#endregion
|
|
452
|
+
//#region ../../packages/sync-engine/src/collections/x.ts
|
|
453
|
+
const xPagination = {
|
|
454
|
+
style: "cursor",
|
|
455
|
+
limitParam: "max_results",
|
|
456
|
+
cursorParam: "pagination_token",
|
|
457
|
+
cursorPath: "meta.next_token",
|
|
458
|
+
itemsPath: "data"
|
|
459
|
+
};
|
|
460
|
+
const defaultDMEventFields = "attachments,created_at,dm_conversation_id,entities,event_type,id,participant_ids,referenced_tweets,sender_id,text";
|
|
461
|
+
const defaultDMExpansions = "attachments.media_keys,participant_ids,referenced_tweets.id,sender_id";
|
|
462
|
+
const defaultDMUserFields = "affiliation,connection_status,created_at,description,entities,id,location,most_recent_tweet_id,name,pinned_tweet_id,profile_banner_url,profile_image_url,protected,public_metrics,receives_your_dm,subscription_type,url,username,verified,verified_type,withheld";
|
|
463
|
+
const asRecord = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
464
|
+
const read = (value) => value === null || value === void 0 ? null : value;
|
|
465
|
+
const isoToEpochMs = (value) => {
|
|
466
|
+
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
467
|
+
if (typeof value !== "string") return null;
|
|
468
|
+
const epochMs = Date.parse(value);
|
|
469
|
+
return Number.isNaN(epochMs) ? null : epochMs;
|
|
470
|
+
};
|
|
471
|
+
const toBool = (value) => {
|
|
472
|
+
if (typeof value !== "boolean") return null;
|
|
473
|
+
return value ? 1 : 0;
|
|
474
|
+
};
|
|
475
|
+
const stringifyJSON$1 = (value) => value === null || value === void 0 ? null : JSON.stringify(value);
|
|
476
|
+
const liftTweetColumns = (item) => {
|
|
477
|
+
const publicMetrics = asRecord(item.public_metrics);
|
|
478
|
+
return {
|
|
479
|
+
id: read(item.id),
|
|
480
|
+
created_at: isoToEpochMs(item.created_at),
|
|
481
|
+
author_id: read(item.author_id),
|
|
482
|
+
conversation_id: read(item.conversation_id),
|
|
483
|
+
in_reply_to_user_id: read(item.in_reply_to_user_id),
|
|
484
|
+
lang: read(item.lang),
|
|
485
|
+
reply_settings: read(item.reply_settings),
|
|
486
|
+
text: read(item.text),
|
|
487
|
+
url: read(item.url),
|
|
488
|
+
paid_partnership: toBool(item.paid_partnership),
|
|
489
|
+
possibly_sensitive: toBool(item.possibly_sensitive),
|
|
490
|
+
like_count: read(publicMetrics.like_count),
|
|
491
|
+
retweet_count: read(publicMetrics.retweet_count),
|
|
492
|
+
reply_count: read(publicMetrics.reply_count),
|
|
493
|
+
quote_count: read(publicMetrics.quote_count),
|
|
494
|
+
bookmark_count: read(publicMetrics.bookmark_count),
|
|
495
|
+
impression_count: read(publicMetrics.impression_count)
|
|
496
|
+
};
|
|
497
|
+
};
|
|
498
|
+
const liftTweet = (item) => ({
|
|
499
|
+
...liftTweetColumns(item),
|
|
500
|
+
raw: JSON.stringify(item)
|
|
501
|
+
});
|
|
502
|
+
const liftBookmark = (item) => {
|
|
503
|
+
const article = asRecord(item.article);
|
|
504
|
+
const noteTweet = asRecord(item.note_tweet);
|
|
505
|
+
return {
|
|
506
|
+
...liftTweetColumns(item),
|
|
507
|
+
article_cover_media: read(article.cover_media),
|
|
508
|
+
article_plain_text: read(article.plain_text),
|
|
509
|
+
article_preview_text: read(article.preview_text),
|
|
510
|
+
article_title: read(article.title),
|
|
511
|
+
note_tweet_text: read(noteTweet.text),
|
|
512
|
+
raw: JSON.stringify(item)
|
|
513
|
+
};
|
|
514
|
+
};
|
|
515
|
+
const liftDMEvent = (item) => ({
|
|
516
|
+
id: read(item.id),
|
|
517
|
+
created_at: isoToEpochMs(item.created_at),
|
|
518
|
+
dm_conversation_id: read(item.dm_conversation_id),
|
|
519
|
+
event_type: read(item.event_type),
|
|
520
|
+
sender_id: read(item.sender_id),
|
|
521
|
+
text: read(item.text),
|
|
522
|
+
participant_ids: stringifyJSON$1(item.participant_ids),
|
|
523
|
+
attachments: stringifyJSON$1(item.attachments),
|
|
524
|
+
entities: stringifyJSON$1(item.entities),
|
|
525
|
+
referenced_tweets: stringifyJSON$1(item.referenced_tweets),
|
|
526
|
+
raw: JSON.stringify(item)
|
|
527
|
+
});
|
|
528
|
+
const liftXUser = (item) => {
|
|
529
|
+
const affiliation = asRecord(item.affiliation);
|
|
530
|
+
const publicMetrics = asRecord(item.public_metrics);
|
|
531
|
+
const subscription = asRecord(item.subscription);
|
|
532
|
+
return {
|
|
533
|
+
id: read(item.id),
|
|
534
|
+
created_at: isoToEpochMs(item.created_at),
|
|
535
|
+
username: read(item.username),
|
|
536
|
+
name: read(item.name),
|
|
537
|
+
description: read(item.description),
|
|
538
|
+
location: read(item.location),
|
|
539
|
+
url: read(item.url),
|
|
540
|
+
is_identity_verified: toBool(item.is_identity_verified),
|
|
541
|
+
protected: toBool(item.protected),
|
|
542
|
+
verified: toBool(item.verified),
|
|
543
|
+
verified_type: read(item.verified_type),
|
|
544
|
+
subscription_type: read(item.subscription_type),
|
|
545
|
+
parody: toBool(item.parody),
|
|
546
|
+
most_recent_tweet_id: read(item.most_recent_tweet_id),
|
|
547
|
+
pinned_tweet_id: read(item.pinned_tweet_id),
|
|
548
|
+
profile_banner_url: read(item.profile_banner_url),
|
|
549
|
+
profile_image_url: read(item.profile_image_url),
|
|
550
|
+
affiliation_badge_url: read(affiliation.badge_url),
|
|
551
|
+
affiliation_description: read(affiliation.description),
|
|
552
|
+
affiliation_url: read(affiliation.url),
|
|
553
|
+
subscription_subscribes_to_you: toBool(subscription.subscribes_to_you),
|
|
554
|
+
followers_count: read(publicMetrics.followers_count),
|
|
555
|
+
following_count: read(publicMetrics.following_count),
|
|
556
|
+
tweet_count: read(publicMetrics.tweet_count),
|
|
557
|
+
listed_count: read(publicMetrics.listed_count),
|
|
558
|
+
like_count: read(publicMetrics.like_count),
|
|
559
|
+
media_count: read(publicMetrics.media_count),
|
|
560
|
+
raw: JSON.stringify(item)
|
|
561
|
+
};
|
|
562
|
+
};
|
|
563
|
+
const X_COLLECTIONS = [
|
|
564
|
+
{
|
|
565
|
+
key: "x_tweets",
|
|
566
|
+
table: "x_tweets_raw",
|
|
567
|
+
platform: "x",
|
|
568
|
+
requests: [{ path: "/users/:id/tweets" }],
|
|
569
|
+
ownIdToken: ":id",
|
|
570
|
+
pagination: xPagination,
|
|
571
|
+
pageSize: 100,
|
|
572
|
+
sinceField: "created_at",
|
|
573
|
+
lift: liftTweet
|
|
574
|
+
},
|
|
575
|
+
{
|
|
576
|
+
key: "x_followers",
|
|
577
|
+
table: "x_followers_raw",
|
|
578
|
+
platform: "x",
|
|
579
|
+
requests: [{ path: "/users/:id/followers" }],
|
|
580
|
+
ownIdToken: ":id",
|
|
581
|
+
requiresFullInitialSync: true,
|
|
582
|
+
pagination: xPagination,
|
|
583
|
+
pageSize: 1e3,
|
|
584
|
+
lift: liftXUser
|
|
585
|
+
},
|
|
586
|
+
{
|
|
587
|
+
key: "x_following",
|
|
588
|
+
table: "x_following_raw",
|
|
589
|
+
platform: "x",
|
|
590
|
+
requests: [{ path: "/users/:id/following" }],
|
|
591
|
+
ownIdToken: ":id",
|
|
592
|
+
requiresFullInitialSync: true,
|
|
593
|
+
pagination: xPagination,
|
|
594
|
+
pageSize: 1e3,
|
|
595
|
+
lift: liftXUser
|
|
596
|
+
},
|
|
597
|
+
{
|
|
598
|
+
key: "x_bookmarks",
|
|
599
|
+
table: "x_bookmarks_raw",
|
|
600
|
+
platform: "x",
|
|
601
|
+
requests: [{ path: "/users/:id/bookmarks" }],
|
|
602
|
+
ownIdToken: ":id",
|
|
603
|
+
pagination: xPagination,
|
|
604
|
+
pageSize: 100,
|
|
605
|
+
lift: liftBookmark
|
|
606
|
+
},
|
|
607
|
+
{
|
|
608
|
+
key: "x_messages",
|
|
609
|
+
table: "x_messages_raw",
|
|
610
|
+
platform: "x",
|
|
611
|
+
requests: [{
|
|
612
|
+
path: "/dm_events",
|
|
613
|
+
bakedQuery: {
|
|
614
|
+
"dm_event.fields": defaultDMEventFields,
|
|
615
|
+
expansions: defaultDMExpansions,
|
|
616
|
+
"user.fields": defaultDMUserFields
|
|
617
|
+
}
|
|
618
|
+
}],
|
|
619
|
+
pagination: xPagination,
|
|
620
|
+
pageSize: 100,
|
|
621
|
+
sinceField: "created_at",
|
|
622
|
+
lift: liftDMEvent
|
|
623
|
+
}
|
|
624
|
+
];
|
|
625
|
+
const X_CACHE_METADATA = {
|
|
626
|
+
joins: ["x_messages.sender_id = x_profiles.provider_id", "x_followers.id = x_profiles.provider_id"],
|
|
627
|
+
enums: {
|
|
628
|
+
"x_followers.verified_type": [
|
|
629
|
+
"blue",
|
|
630
|
+
"business",
|
|
631
|
+
"government",
|
|
632
|
+
"none"
|
|
633
|
+
],
|
|
634
|
+
"x_following.verified_type": [
|
|
635
|
+
"blue",
|
|
636
|
+
"business",
|
|
637
|
+
"government",
|
|
638
|
+
"none"
|
|
639
|
+
]
|
|
640
|
+
},
|
|
641
|
+
indexed: {
|
|
642
|
+
x_bookmarks: [
|
|
643
|
+
"created_at",
|
|
644
|
+
"author_id",
|
|
645
|
+
"conversation_id"
|
|
646
|
+
],
|
|
647
|
+
x_followers: [
|
|
648
|
+
"created_at",
|
|
649
|
+
"username",
|
|
650
|
+
"verified",
|
|
651
|
+
"verified_type"
|
|
652
|
+
],
|
|
653
|
+
x_following: [
|
|
654
|
+
"created_at",
|
|
655
|
+
"username",
|
|
656
|
+
"verified",
|
|
657
|
+
"verified_type"
|
|
658
|
+
],
|
|
659
|
+
x_messages: [
|
|
660
|
+
"created_at",
|
|
661
|
+
"conversation_id",
|
|
662
|
+
"sender_id"
|
|
663
|
+
],
|
|
664
|
+
x_profiles: ["username"],
|
|
665
|
+
x_tweets: [
|
|
666
|
+
"created_at",
|
|
667
|
+
"author_id",
|
|
668
|
+
"conversation_id"
|
|
669
|
+
]
|
|
296
670
|
}
|
|
297
|
-
return tables;
|
|
298
|
-
};
|
|
299
|
-
const assertPlatformQuery = (sql, platform) => {
|
|
300
|
-
const allowedTables = tablesForPlatform(platform);
|
|
301
|
-
const rejectedTables = new Set(knownTableTokens(sql).filter((table) => !allowedTables.has(table)));
|
|
302
|
-
if (rejectedTables.size > 0) throw new Error(`Cache SQL for ${platform} cannot read ${[...rejectedTables].sort().join(", ")}.`);
|
|
303
|
-
};
|
|
304
|
-
const referencedTablesForRead = (sql, platform) => {
|
|
305
|
-
const allowedTables = platformTables[platform];
|
|
306
|
-
return knownTableTokens(sql).filter((table) => allowedTables.has(table));
|
|
307
|
-
};
|
|
308
|
-
const collectionForReadTable = (table) => table === "x_raw_messages" ? "x_messages" : table;
|
|
309
|
-
const collectionShortName = (collection) => collection.replace(/^(x|li)_/, "");
|
|
310
|
-
const formatList = (items) => {
|
|
311
|
-
if (items.length <= 1) return items[0] ?? "";
|
|
312
|
-
if (items.length === 2) return `${items[0]} and ${items[1]}`;
|
|
313
|
-
return `${items.slice(0, -1).join(", ")}, and ${items.at(-1)}`;
|
|
314
671
|
};
|
|
315
|
-
|
|
316
|
-
|
|
317
|
-
|
|
672
|
+
//#endregion
|
|
673
|
+
//#region ../../packages/sync-engine/src/collections/index.ts
|
|
674
|
+
const COLLECTIONS = [...X_COLLECTIONS, ...LINKEDIN_COLLECTIONS];
|
|
675
|
+
const CACHE_METADATA_BY_PLATFORM = {
|
|
676
|
+
linkedin: LINKEDIN_CACHE_METADATA,
|
|
677
|
+
x: X_CACHE_METADATA
|
|
318
678
|
};
|
|
319
|
-
const
|
|
320
|
-
|
|
321
|
-
|
|
679
|
+
const collectionByKey = (key) => COLLECTIONS.find((collection) => collection.key === key);
|
|
680
|
+
const collectionsFor = (platform) => COLLECTIONS.filter((collection) => collection.platform === platform);
|
|
681
|
+
const syncableCollectionsFor = (platform) => {
|
|
682
|
+
const parentKeys = new Set(COLLECTIONS.map((collection) => collection.parentCollection).filter(Boolean));
|
|
683
|
+
return collectionsFor(platform).filter((collection) => !parentKeys.has(collection.key));
|
|
322
684
|
};
|
|
323
|
-
const
|
|
324
|
-
const
|
|
325
|
-
|
|
326
|
-
|
|
327
|
-
|
|
328
|
-
assertAtLatest(db);
|
|
329
|
-
return createCache(dbPath, db);
|
|
330
|
-
}
|
|
331
|
-
if (dbPath !== ":memory:") mkdirSync(dirname(dbPath), { recursive: true });
|
|
332
|
-
const db = openDatabase(dbPath);
|
|
333
|
-
const result = migrate(db);
|
|
334
|
-
if (result.applied > 0) opts?.onMigrate?.(result);
|
|
335
|
-
return createCache(dbPath, db);
|
|
685
|
+
const physicalTableFor = (collection) => collection.table ?? collection.key;
|
|
686
|
+
const resetTablesFor = (collection) => {
|
|
687
|
+
if (!collection.parentCollection) return [collection];
|
|
688
|
+
const parent = collectionByKey(collection.parentCollection);
|
|
689
|
+
return parent ? [collection, parent] : [collection];
|
|
336
690
|
};
|
|
337
|
-
const
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
assertKnownTable(table);
|
|
345
|
-
const columns = [...Object.keys(rows[0] ?? {}).filter((column) => column !== "synced_at"), "synced_at"];
|
|
346
|
-
const updateColumns = columns.filter((column) => column !== "id");
|
|
347
|
-
const statement = db.prepare(`INSERT INTO ${quoteIdentifier(table)} (${columns.map(quoteIdentifier).join(", ")}) VALUES (${columns.map((column) => `:${column}`).join(", ")}) ON CONFLICT(id) DO UPDATE SET ${updateColumns.map((column) => `${quoteIdentifier(column)}=excluded.${quoteIdentifier(column)}`).join(", ")}`);
|
|
348
|
-
db.transaction(() => {
|
|
349
|
-
for (const row of rows) statement.run({
|
|
350
|
-
...row,
|
|
351
|
-
synced_at: syncedAt
|
|
352
|
-
});
|
|
353
|
-
});
|
|
354
|
-
},
|
|
355
|
-
upsertProfiles: (table, rows) => {
|
|
356
|
-
if (rows.length === 0) return;
|
|
357
|
-
assertKnownProfileTable(table);
|
|
358
|
-
const rowsByProviderId = /* @__PURE__ */ new Map();
|
|
359
|
-
for (const row of rows) rowsByProviderId.set(row.providerId, row);
|
|
360
|
-
const columns = [
|
|
361
|
-
"provider_id",
|
|
362
|
-
"handle",
|
|
363
|
-
"display_name",
|
|
364
|
-
"avatar_url",
|
|
365
|
-
"headline",
|
|
366
|
-
"raw",
|
|
367
|
-
"synced_at"
|
|
368
|
-
];
|
|
369
|
-
const quotedTable = quoteIdentifier(table);
|
|
370
|
-
const nullableColumns = [
|
|
371
|
-
"handle",
|
|
372
|
-
"avatar_url",
|
|
373
|
-
"headline"
|
|
374
|
-
];
|
|
375
|
-
const lastWriteColumns = [
|
|
376
|
-
"display_name",
|
|
377
|
-
"raw",
|
|
378
|
-
"synced_at"
|
|
379
|
-
];
|
|
380
|
-
const updateAssignments = [...nullableColumns.map((column) => `${quoteIdentifier(column)}=COALESCE(excluded.${quoteIdentifier(column)}, ${quotedTable}.${quoteIdentifier(column)})`), ...lastWriteColumns.map((column) => `${quoteIdentifier(column)}=excluded.${quoteIdentifier(column)}`)].join(", ");
|
|
381
|
-
const statement = db.prepare(`INSERT INTO ${quotedTable} (${columns.map(quoteIdentifier).join(", ")}) VALUES (${columns.map((column) => `:${column}`).join(", ")}) ON CONFLICT(provider_id) DO UPDATE SET ${updateAssignments}`);
|
|
382
|
-
const syncedAt = Date.now();
|
|
383
|
-
db.transaction(() => {
|
|
384
|
-
for (const row of rowsByProviderId.values()) statement.run({
|
|
385
|
-
provider_id: row.providerId,
|
|
386
|
-
handle: row.handle,
|
|
387
|
-
display_name: row.displayName,
|
|
388
|
-
avatar_url: row.avatarURL,
|
|
389
|
-
headline: row.headline,
|
|
390
|
-
raw: JSON.stringify(row.raw),
|
|
391
|
-
synced_at: syncedAt
|
|
392
|
-
});
|
|
393
|
-
});
|
|
691
|
+
const resetPhysicalTablesFor = (collection) => resetTablesFor(collection).map(physicalTableFor);
|
|
692
|
+
//#endregion
|
|
693
|
+
//#region ../../packages/sync-engine/src/migrations.generated.ts
|
|
694
|
+
const MIGRATIONS = [
|
|
695
|
+
{
|
|
696
|
+
"tag": "0000_modern_stephen_strange",
|
|
697
|
+
"sql": "CREATE TABLE `li_connections` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `object` text,\n `url` text,\n `user_id` text,\n `user_object` text,\n `user_type` text,\n `user_display_name` text,\n `user_public_identifier` text,\n `user_profile_url` text,\n `user_public_picture_url` text,\n `user_description` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `li_connections_created_at_idx` ON `li_connections` (`created_at`);--> statement-breakpoint\nCREATE TABLE `li_conversations` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `last_message_timestamp` integer,\n `muted_until` integer,\n `creator_id` text,\n `headline` text,\n `image_url` text,\n `name` text,\n `object` text,\n `provider` text,\n `type` text,\n `user_id` text,\n `is_1to1` integer,\n `is_archived` integer,\n `is_channel` integer,\n `is_group` integer,\n `is_pinned` integer,\n `is_readonly` integer,\n `unread_count` integer,\n `user_object` text,\n `user_type` text,\n `user_display_name` text,\n `user_public_identifier` text,\n `user_profile_url` text,\n `user_public_picture_url` text,\n `user_description` text,\n `user_specifics_network_distance` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `li_conversations_created_at_idx` ON `li_conversations` (`created_at`);--> statement-breakpoint\nCREATE INDEX `li_conversations_last_message_timestamp_idx` ON `li_conversations` (`last_message_timestamp`);--> statement-breakpoint\nCREATE TABLE `li_messages` (\n `id` text PRIMARY KEY NOT NULL,\n `timestamp` integer,\n `chat_id` text,\n `object` text,\n `provider` text,\n `text` text,\n `sender_id` text,\n `is_deleted` integer,\n `is_delivered` integer,\n `is_edited` integer,\n `is_event` integer,\n `is_hidden` integer,\n `is_mentionned` integer,\n `is_pinned` integer,\n `is_seen` integer,\n `is_sender` integer,\n `sender_object` text,\n `sender_type` text,\n `sender_display_name` text,\n `sender_public_identifier` text,\n `sender_profile_url` text,\n `sender_public_picture_url` text,\n `sender_description` text,\n `sender_specifics_network_distance` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `li_messages_timestamp_idx` ON `li_messages` (`timestamp`);--> statement-breakpoint\nCREATE INDEX `li_messages_chat_id_idx` ON `li_messages` (`chat_id`);--> statement-breakpoint\nCREATE TABLE `li_posts` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `object` text,\n `share_url` text,\n `text` text,\n `url` text,\n `is_repost` integer,\n `user_reacted` text,\n `comments_counter` integer,\n `reposts_counter` integer,\n `analytics_impressions_counter` integer,\n `author_id` text,\n `author_object` text,\n `author_type` text,\n `author_display_name` text,\n `author_public_identifier` text,\n `author_profile_url` text,\n `author_public_picture_url` text,\n `author_description` text,\n `author_specifics_network_distance` text,\n `permissions_can_post_comments` integer,\n `permissions_can_react` integer,\n `permissions_can_share` integer,\n `poll_id` text,\n `poll_question` text,\n `poll_is_open` integer,\n `poll_total_votes_count` integer,\n `quoted_post_id` text,\n `quoted_post_object` text,\n `quoted_post_created_at` integer,\n `quoted_post_text` text,\n `quoted_post_share_url` text,\n `quoted_post_user_reacted` text,\n `quoted_post_comments_counter` integer,\n `quoted_post_reposts_counter` integer,\n `quoted_post_permissions_can_post_comments` integer,\n `quoted_post_permissions_can_react` integer,\n `quoted_post_permissions_can_share` integer,\n `quoted_post_author_id` text,\n `quoted_post_author_object` text,\n `quoted_post_author_type` text,\n `quoted_post_author_display_name` text,\n `quoted_post_author_public_identifier` text,\n `quoted_post_author_profile_url` text,\n `quoted_post_author_public_picture_url` text,\n `quoted_post_author_description` text,\n `quoted_post_author_specifics_network_distance` text,\n `reposted_by_id` text,\n `reposted_by_object` text,\n `reposted_by_type` text,\n `reposted_by_display_name` text,\n `reposted_by_public_identifier` text,\n `reposted_by_profile_url` text,\n `reposted_by_public_picture_url` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `li_posts_created_at_idx` ON `li_posts` (`created_at`);--> statement-breakpoint\nCREATE TABLE `li_requests` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `object` text,\n `type` text,\n `user_id` text,\n `user_object` text,\n `user_type` text,\n `user_display_name` text,\n `user_public_identifier` text,\n `user_profile_url` text,\n `user_public_picture_url` text,\n `user_description` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `li_requests_created_at_idx` ON `li_requests` (`created_at`);--> statement-breakpoint\nCREATE TABLE `sync_state` (\n `collection` text PRIMARY KEY NOT NULL,\n `cursor` text,\n `last_synced` integer,\n `object_count` integer\n);\n--> statement-breakpoint\nCREATE TABLE `x_bookmarks` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `author_id` text,\n `conversation_id` text,\n `in_reply_to_user_id` text,\n `lang` text,\n `reply_settings` text,\n `text` text,\n `url` text,\n `paid_partnership` integer,\n `possibly_sensitive` integer,\n `edit_controls_editable_until` integer,\n `edit_controls_edits_remaining` integer,\n `edit_controls_is_edit_eligible` integer,\n `like_count` integer,\n `retweet_count` integer,\n `reply_count` integer,\n `quote_count` integer,\n `bookmark_count` integer,\n `impression_count` integer,\n `article_cover_media` text,\n `article_plain_text` text,\n `article_preview_text` text,\n `article_title` text,\n `note_tweet_text` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `x_bookmarks_created_at_idx` ON `x_bookmarks` (`created_at`);--> statement-breakpoint\nCREATE TABLE `x_followers` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `username` text,\n `name` text,\n `description` text,\n `location` text,\n `url` text,\n `is_identity_verified` integer,\n `protected` integer,\n `verified` integer,\n `verified_type` text,\n `subscription_type` text,\n `parody` integer,\n `most_recent_tweet_id` text,\n `pinned_tweet_id` text,\n `profile_banner_url` text,\n `profile_image_url` text,\n `affiliation_badge_url` text,\n `affiliation_description` text,\n `affiliation_url` text,\n `subscription_subscribes_to_you` integer,\n `followers_count` integer,\n `following_count` integer,\n `tweet_count` integer,\n `listed_count` integer,\n `like_count` integer,\n `media_count` integer,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `x_followers_created_at_idx` ON `x_followers` (`created_at`);--> statement-breakpoint\nCREATE TABLE `x_following` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `username` text,\n `name` text,\n `description` text,\n `location` text,\n `url` text,\n `is_identity_verified` integer,\n `protected` integer,\n `verified` integer,\n `verified_type` text,\n `subscription_type` text,\n `parody` integer,\n `most_recent_tweet_id` text,\n `pinned_tweet_id` text,\n `profile_banner_url` text,\n `profile_image_url` text,\n `affiliation_badge_url` text,\n `affiliation_description` text,\n `affiliation_url` text,\n `subscription_subscribes_to_you` integer,\n `followers_count` integer,\n `following_count` integer,\n `tweet_count` integer,\n `listed_count` integer,\n `like_count` integer,\n `media_count` integer,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `x_following_created_at_idx` ON `x_following` (`created_at`);--> statement-breakpoint\nCREATE TABLE `x_tweets` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `author_id` text,\n `conversation_id` text,\n `in_reply_to_user_id` text,\n `lang` text,\n `reply_settings` text,\n `text` text,\n `url` text,\n `paid_partnership` integer,\n `possibly_sensitive` integer,\n `edit_controls_editable_until` integer,\n `edit_controls_edits_remaining` integer,\n `edit_controls_is_edit_eligible` integer,\n `like_count` integer,\n `retweet_count` integer,\n `reply_count` integer,\n `quote_count` integer,\n `bookmark_count` integer,\n `impression_count` integer,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `x_tweets_created_at_idx` ON `x_tweets` (`created_at`);"
|
|
394
698
|
},
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
return {
|
|
399
|
-
cursor: stringOrNull(row.cursor),
|
|
400
|
-
lastSynced: numberOrNull(row.last_synced),
|
|
401
|
-
objectCount: numberOrNull(row.object_count)
|
|
402
|
-
};
|
|
699
|
+
{
|
|
700
|
+
"tag": "0001_far_beast",
|
|
701
|
+
"sql": "CREATE TABLE `li_profiles` (\n `provider_id` text PRIMARY KEY NOT NULL,\n `handle` text,\n `display_name` text NOT NULL,\n `avatar_url` text,\n `headline` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `li_profiles_handle_idx` ON `li_profiles` (`handle`);--> statement-breakpoint\nCREATE TABLE `x_messages` (\n `id` text PRIMARY KEY NOT NULL,\n `created_at` integer,\n `dm_conversation_id` text,\n `event_type` text,\n `sender_id` text,\n `text` text,\n `participant_ids` text,\n `attachments` text,\n `entities` text,\n `referenced_tweets` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `x_messages_created_at_idx` ON `x_messages` (`created_at`);--> statement-breakpoint\nCREATE INDEX `x_messages_dm_conversation_id_idx` ON `x_messages` (`dm_conversation_id`);--> statement-breakpoint\nCREATE TABLE `x_profiles` (\n `provider_id` text PRIMARY KEY NOT NULL,\n `handle` text,\n `display_name` text NOT NULL,\n `avatar_url` text,\n `headline` text,\n `raw` text NOT NULL,\n `synced_at` integer NOT NULL\n);\n--> statement-breakpoint\nCREATE INDEX `x_profiles_handle_idx` ON `x_profiles` (`handle`);"
|
|
403
702
|
},
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
cursor,
|
|
408
|
-
last_synced: lastSynced,
|
|
409
|
-
object_count: objectCount
|
|
410
|
-
});
|
|
703
|
+
{
|
|
704
|
+
"tag": "0002_previous_spitfire",
|
|
705
|
+
"sql": "ALTER TABLE `li_connections` ADD `user_headline` text;--> statement-breakpoint\nALTER TABLE `li_requests` ADD `user_headline` text;--> statement-breakpoint\nALTER TABLE `x_bookmarks` DROP COLUMN `edit_controls_editable_until`;--> statement-breakpoint\nALTER TABLE `x_bookmarks` DROP COLUMN `edit_controls_edits_remaining`;--> statement-breakpoint\nALTER TABLE `x_bookmarks` DROP COLUMN `edit_controls_is_edit_eligible`;--> statement-breakpoint\nALTER TABLE `x_tweets` DROP COLUMN `edit_controls_editable_until`;--> statement-breakpoint\nALTER TABLE `x_tweets` DROP COLUMN `edit_controls_edits_remaining`;--> statement-breakpoint\nALTER TABLE `x_tweets` DROP COLUMN `edit_controls_is_edit_eligible`;"
|
|
411
706
|
},
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
db.transaction(() => {
|
|
416
|
-
for (const table of tables) {
|
|
417
|
-
const result = db.prepare(`DELETE FROM ${quoteIdentifier(table)}`).run();
|
|
418
|
-
results.push({
|
|
419
|
-
table,
|
|
420
|
-
rowsDeleted: result?.changes ?? 0
|
|
421
|
-
});
|
|
422
|
-
}
|
|
423
|
-
const deleteState = db.prepare("DELETE FROM sync_state WHERE collection = :collection");
|
|
424
|
-
for (const collection of collections) deleteState.run({ collection });
|
|
425
|
-
});
|
|
426
|
-
return results;
|
|
707
|
+
{
|
|
708
|
+
"tag": "0003_wonderful_makkari",
|
|
709
|
+
"sql": "ALTER TABLE `li_conversations` RENAME COLUMN \"headline\" TO \"user_headline\";--> statement-breakpoint\nCREATE INDEX `li_conversations_user_id_last_message_timestamp_idx` ON `li_conversations` (`user_id`,`last_message_timestamp`);--> statement-breakpoint\nCREATE INDEX `li_conversations_user_public_identifier_last_message_timestamp_idx` ON `li_conversations` (`user_public_identifier`,`last_message_timestamp`);--> statement-breakpoint\nCREATE INDEX `li_conversations_user_display_name_idx` ON `li_conversations` (`user_display_name`);--> statement-breakpoint\nCREATE INDEX `li_conversations_is_archived_last_message_timestamp_idx` ON `li_conversations` (`is_archived`,`last_message_timestamp`);--> statement-breakpoint\nDROP INDEX `li_messages_chat_id_idx`;--> statement-breakpoint\nCREATE INDEX `li_messages_chat_id_timestamp_idx` ON `li_messages` (`chat_id`,`timestamp`);--> statement-breakpoint\nCREATE INDEX `li_messages_sender_id_timestamp_idx` ON `li_messages` (`sender_id`,`timestamp`);--> statement-breakpoint\nDROP INDEX `x_messages_dm_conversation_id_idx`;--> statement-breakpoint\nCREATE INDEX `x_messages_dm_conversation_id_created_at_idx` ON `x_messages` (`dm_conversation_id`,`created_at`);--> statement-breakpoint\nCREATE INDEX `x_messages_sender_id_created_at_idx` ON `x_messages` (`sender_id`,`created_at`);--> statement-breakpoint\nCREATE INDEX `li_connections_synced_at_idx` ON `li_connections` (`synced_at`);--> statement-breakpoint\nCREATE INDEX `li_connections_user_id_idx` ON `li_connections` (`user_id`);--> statement-breakpoint\nCREATE INDEX `li_connections_user_public_identifier_idx` ON `li_connections` (`user_public_identifier`);--> statement-breakpoint\nCREATE INDEX `li_connections_user_display_name_idx` ON `li_connections` (`user_display_name`);--> statement-breakpoint\nCREATE INDEX `li_posts_author_id_created_at_idx` ON `li_posts` (`author_id`,`created_at`);--> statement-breakpoint\nCREATE INDEX `li_posts_author_public_identifier_created_at_idx` ON `li_posts` (`author_public_identifier`,`created_at`);--> statement-breakpoint\nCREATE INDEX `li_requests_type_synced_at_idx` ON `li_requests` (`type`,`synced_at`);--> statement-breakpoint\nCREATE INDEX `li_requests_user_id_idx` ON `li_requests` (`user_id`);--> statement-breakpoint\nCREATE INDEX `li_requests_user_public_identifier_idx` ON `li_requests` (`user_public_identifier`);--> statement-breakpoint\nCREATE INDEX `li_requests_user_display_name_idx` ON `li_requests` (`user_display_name`);--> statement-breakpoint\nCREATE INDEX `x_bookmarks_author_id_created_at_idx` ON `x_bookmarks` (`author_id`,`created_at`);--> statement-breakpoint\nCREATE INDEX `x_bookmarks_conversation_id_created_at_idx` ON `x_bookmarks` (`conversation_id`,`created_at`);--> statement-breakpoint\nCREATE INDEX `x_followers_synced_at_idx` ON `x_followers` (`synced_at`);--> statement-breakpoint\nCREATE INDEX `x_followers_username_idx` ON `x_followers` (`username`);--> statement-breakpoint\nCREATE INDEX `x_followers_verified_idx` ON `x_followers` (`verified`);--> statement-breakpoint\nCREATE INDEX `x_followers_verified_type_idx` ON `x_followers` (`verified_type`);--> statement-breakpoint\nCREATE INDEX `x_following_synced_at_idx` ON `x_following` (`synced_at`);--> statement-breakpoint\nCREATE INDEX `x_following_username_idx` ON `x_following` (`username`);--> statement-breakpoint\nCREATE INDEX `x_following_verified_idx` ON `x_following` (`verified`);--> statement-breakpoint\nCREATE INDEX `x_following_verified_type_idx` ON `x_following` (`verified_type`);--> statement-breakpoint\nCREATE INDEX `x_tweets_author_id_created_at_idx` ON `x_tweets` (`author_id`,`created_at`);--> statement-breakpoint\nCREATE INDEX `x_tweets_conversation_id_created_at_idx` ON `x_tweets` (`conversation_id`,`created_at`);"
|
|
427
710
|
},
|
|
428
|
-
|
|
429
|
-
|
|
430
|
-
|
|
711
|
+
{
|
|
712
|
+
"tag": "0004_quiet_prism",
|
|
713
|
+
"sql": "ALTER TABLE `x_messages` RENAME TO `x_raw_messages`;--> statement-breakpoint\nCREATE VIEW `x_messages` AS\nSELECT m.*, p.handle AS sender_handle, p.display_name AS sender_name,\n p.avatar_url AS sender_avatar_url, p.headline AS sender_headline\nFROM `x_raw_messages` m LEFT JOIN `x_profiles` p ON m.sender_id = p.provider_id;\n"
|
|
431
714
|
},
|
|
432
|
-
|
|
433
|
-
|
|
434
|
-
|
|
435
|
-
const collection = collectionForReadTable(name);
|
|
436
|
-
return {
|
|
437
|
-
name,
|
|
438
|
-
collection,
|
|
439
|
-
lastSynced: numberOrNull(db.prepare("SELECT last_synced FROM sync_state WHERE collection = :collection").get({ collection })?.last_synced)
|
|
440
|
-
};
|
|
441
|
-
});
|
|
442
|
-
const missingCollections = [...new Set(tables.filter((table) => table.lastSynced == null).map((table) => table.collection))];
|
|
443
|
-
if (missingCollections.length > 0) throw new NeverSyncedError(missingCollections, neverSyncedMessage(opts.platform, missingCollections));
|
|
444
|
-
return {
|
|
445
|
-
rows: db.prepare(sql).all(),
|
|
446
|
-
tables: tables.map(({ name, lastSynced }) => ({
|
|
447
|
-
name,
|
|
448
|
-
lastSynced
|
|
449
|
-
}))
|
|
450
|
-
};
|
|
715
|
+
{
|
|
716
|
+
"tag": "0005_raw_table_views",
|
|
717
|
+
"sql": "DROP VIEW IF EXISTS \"x_messages\";\nALTER TABLE \"x_tweets\" RENAME TO \"x_tweets_raw\";\nALTER TABLE \"x_bookmarks\" RENAME TO \"x_bookmarks_raw\";\nALTER TABLE \"x_followers\" RENAME TO \"x_followers_raw\";\nALTER TABLE \"x_following\" RENAME TO \"x_following_raw\";\nALTER TABLE \"x_profiles\" RENAME TO \"x_profiles_raw\";\nALTER TABLE \"x_raw_messages\" RENAME TO \"x_messages_raw\";\nALTER TABLE \"li_connections\" RENAME TO \"li_connections_raw\";\nALTER TABLE \"li_profiles\" RENAME TO \"li_profiles_raw\";\nALTER TABLE \"li_conversations\" RENAME TO \"li_conversations_raw\";\nALTER TABLE \"li_messages\" RENAME TO \"li_messages_raw\";\nALTER TABLE \"li_posts\" RENAME TO \"li_posts_raw\";\nALTER TABLE \"li_requests\" RENAME TO \"li_requests_raw\";\nCREATE VIEW \"x_tweets\" AS\nSELECT \"id\", \"created_at\", \"author_id\", \"conversation_id\", \"in_reply_to_user_id\", \"lang\",\n \"reply_settings\", \"text\", \"url\", \"paid_partnership\", \"possibly_sensitive\",\n \"like_count\", \"retweet_count\", \"reply_count\", \"quote_count\", \"bookmark_count\",\n \"impression_count\"\nFROM \"x_tweets_raw\";\nCREATE VIEW \"x_bookmarks\" AS\nSELECT \"id\", \"created_at\", \"author_id\", \"conversation_id\", \"in_reply_to_user_id\", \"lang\",\n \"reply_settings\", \"text\", \"url\", \"paid_partnership\", \"possibly_sensitive\",\n \"like_count\", \"retweet_count\", \"reply_count\", \"quote_count\", \"bookmark_count\",\n \"impression_count\", \"article_cover_media\", \"article_plain_text\",\n \"article_preview_text\", \"article_title\", \"note_tweet_text\"\nFROM \"x_bookmarks_raw\";\nCREATE VIEW \"x_followers\" AS\nSELECT \"id\", \"created_at\", \"username\", \"name\", \"description\", \"location\", \"url\",\n \"verified\", \"verified_type\", \"is_identity_verified\", \"protected\",\n \"subscription_type\", \"parody\", \"followers_count\", \"following_count\",\n \"tweet_count\", \"listed_count\", \"like_count\", \"media_count\", \"most_recent_tweet_id\",\n \"pinned_tweet_id\", \"profile_image_url\" AS \"avatar_url\", \"profile_banner_url\",\n \"affiliation_description\", \"affiliation_url\", \"affiliation_badge_url\",\n \"subscription_subscribes_to_you\"\nFROM \"x_followers_raw\";\nCREATE VIEW \"x_following\" AS\nSELECT \"id\", \"created_at\", \"username\", \"name\", \"description\", \"location\", \"url\",\n \"verified\", \"verified_type\", \"is_identity_verified\", \"protected\",\n \"subscription_type\", \"parody\", \"followers_count\", \"following_count\",\n \"tweet_count\", \"listed_count\", \"like_count\", \"media_count\", \"most_recent_tweet_id\",\n \"pinned_tweet_id\", \"profile_image_url\" AS \"avatar_url\", \"profile_banner_url\",\n \"affiliation_description\", \"affiliation_url\", \"affiliation_badge_url\",\n \"subscription_subscribes_to_you\"\nFROM \"x_following_raw\";\nCREATE VIEW \"x_messages\" AS\nSELECT m.\"id\", m.\"created_at\", m.\"dm_conversation_id\" AS \"conversation_id\",\n m.\"event_type\", m.\"sender_id\", m.\"text\", m.\"participant_ids\", m.\"attachments\",\n m.\"entities\", m.\"referenced_tweets\", p.\"handle\" AS \"sender_username\",\n p.\"display_name\" AS \"sender_name\", p.\"avatar_url\" AS \"sender_avatar_url\",\n p.\"headline\" AS \"sender_headline\"\nFROM \"x_messages_raw\" m\nLEFT JOIN \"x_profiles_raw\" p ON m.\"sender_id\" = p.\"provider_id\";\nCREATE VIEW \"x_profiles\" AS\nSELECT \"provider_id\", \"handle\" AS \"username\", \"display_name\" AS \"name\", \"avatar_url\",\n \"headline\", json_extract(\"raw\", '$.receives_your_dm') AS \"receives_your_dm\"\nFROM \"x_profiles_raw\";\nCREATE VIEW \"li_profiles\" AS\nSELECT \"provider_id\", \"handle\" AS \"username\", \"display_name\" AS \"name\", \"avatar_url\",\n \"headline\"\nFROM \"li_profiles_raw\";\nCREATE VIEW \"li_connections\" AS\nSELECT \"id\", \"created_at\", \"url\", \"user_id\", \"user_public_identifier\" AS \"user_username\",\n \"user_display_name\" AS \"user_name\", \"user_headline\", \"user_description\",\n \"user_profile_url\", \"user_public_picture_url\" AS \"user_avatar_url\", \"user_type\"\nFROM \"li_connections_raw\";\nCREATE VIEW \"li_requests\" AS\nSELECT \"id\", \"created_at\", \"type\", json_extract(\"raw\", '$.message') AS \"message\",\n \"user_id\", \"user_public_identifier\" AS \"user_username\",\n \"user_display_name\" AS \"user_name\", \"user_headline\", \"user_description\",\n \"user_profile_url\", \"user_public_picture_url\" AS \"user_avatar_url\", \"user_type\"\nFROM \"li_requests_raw\";\nCREATE VIEW \"li_conversations\" AS\nSELECT \"id\", \"created_at\", \"last_message_timestamp\" AS \"last_message_at\", \"name\",\n CASE\n WHEN \"is_1to1\" = 1 THEN '1to1'\n WHEN \"is_group\" = 1 THEN 'group'\n ELSE \"type\"\n END AS \"type\",\n \"unread_count\", \"is_archived\", \"is_pinned\", \"is_readonly\", \"is_channel\",\n \"muted_until\", \"image_url\", \"creator_id\", \"user_id\",\n \"user_public_identifier\" AS \"user_username\", \"user_display_name\" AS \"user_name\",\n \"user_headline\", \"user_description\", \"user_profile_url\",\n \"user_public_picture_url\" AS \"user_avatar_url\", \"user_type\",\n \"user_specifics_network_distance\" AS \"user_distance\"\nFROM \"li_conversations_raw\";\nCREATE VIEW \"li_messages\" AS\nSELECT \"id\", \"timestamp\" AS \"created_at\", \"chat_id\" AS \"conversation_id\", \"text\",\n \"sender_id\", \"is_deleted\", \"is_delivered\", \"is_edited\", \"is_event\", \"is_hidden\",\n \"is_mentionned\" AS \"is_mentioned\", \"is_pinned\", \"is_seen\", \"is_sender\",\n \"sender_public_identifier\" AS \"sender_username\",\n \"sender_display_name\" AS \"sender_name\", \"sender_description\" AS \"sender_headline\",\n \"sender_public_picture_url\" AS \"sender_avatar_url\", \"sender_profile_url\",\n \"sender_type\", \"sender_specifics_network_distance\" AS \"sender_distance\"\nFROM \"li_messages_raw\";\nCREATE VIEW \"li_posts\" AS\nSELECT \"id\", \"created_at\", \"text\", \"url\", \"share_url\", \"is_repost\", \"user_reacted\",\n \"comments_counter\", \"reposts_counter\",\n \"analytics_impressions_counter\" AS \"impressions_counter\", \"author_id\",\n \"author_public_identifier\" AS \"author_username\",\n \"author_display_name\" AS \"author_name\", \"author_description\", \"author_profile_url\",\n \"author_public_picture_url\" AS \"author_avatar_url\", \"author_type\",\n \"author_specifics_network_distance\" AS \"author_distance\", \"poll_id\", \"poll_question\",\n \"poll_is_open\", \"poll_total_votes_count\", \"quoted_post_id\", \"quoted_post_created_at\",\n \"quoted_post_text\", \"quoted_post_author_public_identifier\" AS \"quoted_post_author_username\",\n \"quoted_post_author_display_name\" AS \"quoted_post_author_name\", \"reposted_by_id\",\n \"reposted_by_public_identifier\" AS \"reposted_by_username\",\n \"reposted_by_display_name\" AS \"reposted_by_name\"\nFROM \"li_posts_raw\";\n"
|
|
451
718
|
},
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
path,
|
|
456
|
-
tables: db.prepare("SELECT name, type, sql FROM sqlite_master WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' ORDER BY name").all().filter((table) => String(table.name) !== "x_raw_messages").filter((table) => allowedTables === void 0 || allowedTables.has(String(table.name))).map((table) => {
|
|
457
|
-
const name = String(table.name);
|
|
458
|
-
const type = table.type === "view" ? "view" : "table";
|
|
459
|
-
const rowCount = db.prepare(`SELECT count(*) AS rows FROM ${quoteIdentifier(name)}`).get();
|
|
460
|
-
const collection = collectionForReadTable(name);
|
|
461
|
-
const syncState = name === "sync_state" ? void 0 : db.prepare("SELECT last_synced FROM sync_state WHERE collection = :collection").get({ collection });
|
|
462
|
-
return {
|
|
463
|
-
name,
|
|
464
|
-
type,
|
|
465
|
-
sql: String(table.sql),
|
|
466
|
-
rows: Number(rowCount?.rows ?? 0),
|
|
467
|
-
last_synced: numberOrNull(syncState?.last_synced)
|
|
468
|
-
};
|
|
469
|
-
})
|
|
470
|
-
};
|
|
719
|
+
{
|
|
720
|
+
"tag": "0006_username_profiles",
|
|
721
|
+
"sql": "DROP VIEW IF EXISTS \"x_messages\";\nDROP VIEW IF EXISTS \"x_profiles\";\nDROP VIEW IF EXISTS \"li_profiles\";\nALTER TABLE \"x_profiles_raw\" RENAME COLUMN \"handle\" TO \"username\";\nALTER TABLE \"li_profiles_raw\" RENAME COLUMN \"handle\" TO \"username\";\nDROP INDEX IF EXISTS \"x_profiles_handle_idx\";\nDROP INDEX IF EXISTS \"li_profiles_handle_idx\";\nCREATE INDEX \"x_profiles_username_idx\" ON \"x_profiles_raw\" (\"username\");\nCREATE INDEX \"li_profiles_username_idx\" ON \"li_profiles_raw\" (\"username\");\nCREATE VIEW \"x_messages\" AS\nSELECT m.\"id\", m.\"created_at\", m.\"dm_conversation_id\" AS \"conversation_id\",\n m.\"event_type\", m.\"sender_id\", m.\"text\", m.\"participant_ids\", m.\"attachments\",\n m.\"entities\", m.\"referenced_tweets\", p.\"username\" AS \"sender_username\",\n p.\"display_name\" AS \"sender_name\", p.\"avatar_url\" AS \"sender_avatar_url\",\n p.\"headline\" AS \"sender_headline\"\nFROM \"x_messages_raw\" m\nLEFT JOIN \"x_profiles_raw\" p ON m.\"sender_id\" = p.\"provider_id\";\nCREATE VIEW \"x_profiles\" AS\nSELECT \"provider_id\", \"username\", \"display_name\" AS \"name\", \"avatar_url\",\n \"headline\", json_extract(\"raw\", '$.receives_your_dm') AS \"receives_your_dm\"\nFROM \"x_profiles_raw\";\nCREATE VIEW \"li_profiles\" AS\nSELECT \"provider_id\", \"username\", \"display_name\" AS \"name\", \"avatar_url\", \"headline\"\nFROM \"li_profiles_raw\";\n"
|
|
471
722
|
},
|
|
472
|
-
|
|
473
|
-
|
|
723
|
+
{
|
|
724
|
+
"tag": "0007_connection_headline_coalesce",
|
|
725
|
+
"sql": "DROP VIEW IF EXISTS \"li_connections\";\nDROP VIEW IF EXISTS \"li_requests\";\nDROP VIEW IF EXISTS \"li_conversations\";\nCREATE VIEW \"li_connections\" AS\nSELECT \"id\", \"created_at\", \"url\", \"user_id\", \"user_public_identifier\" AS \"user_username\",\n \"user_display_name\" AS \"user_name\",\n COALESCE(\"user_headline\", \"user_description\") AS \"user_headline\", \"user_description\",\n \"user_profile_url\", \"user_public_picture_url\" AS \"user_avatar_url\", \"user_type\"\nFROM \"li_connections_raw\";\nCREATE VIEW \"li_requests\" AS\nSELECT \"id\", \"created_at\", \"type\", json_extract(\"raw\", '$.message') AS \"message\",\n \"user_id\", \"user_public_identifier\" AS \"user_username\",\n \"user_display_name\" AS \"user_name\",\n COALESCE(\"user_headline\", \"user_description\") AS \"user_headline\", \"user_description\",\n \"user_profile_url\", \"user_public_picture_url\" AS \"user_avatar_url\", \"user_type\"\nFROM \"li_requests_raw\";\nCREATE VIEW \"li_conversations\" AS\nSELECT \"id\", \"created_at\", \"last_message_timestamp\" AS \"last_message_at\", \"name\",\n CASE\n WHEN \"is_1to1\" = 1 THEN '1to1'\n WHEN \"is_group\" = 1 THEN 'group'\n ELSE \"type\"\n END AS \"type\",\n \"unread_count\", \"is_archived\", \"is_pinned\", \"is_readonly\", \"is_channel\",\n \"muted_until\", \"image_url\", \"creator_id\", \"user_id\",\n \"user_public_identifier\" AS \"user_username\", \"user_display_name\" AS \"user_name\",\n COALESCE(\"user_headline\", \"user_description\") AS \"user_headline\", \"user_description\",\n \"user_profile_url\", \"user_public_picture_url\" AS \"user_avatar_url\", \"user_type\",\n \"user_specifics_network_distance\" AS \"user_distance\"\nFROM \"li_conversations_raw\";\n"
|
|
474
726
|
}
|
|
475
|
-
|
|
727
|
+
];
|
|
476
728
|
//#endregion
|
|
477
|
-
//#region ../../packages/sync-engine/src/
|
|
478
|
-
const
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
cursorParam: "cursor",
|
|
482
|
-
cursorPath: "next_cursor",
|
|
483
|
-
totalCountPath: "total_count",
|
|
484
|
-
itemsPath: "data"
|
|
729
|
+
//#region ../../packages/sync-engine/src/migrate.ts
|
|
730
|
+
const readUserVersion = (db) => {
|
|
731
|
+
const version = db.prepare("PRAGMA user_version").get()?.user_version;
|
|
732
|
+
return typeof version === "number" ? version : Number(version ?? 0);
|
|
485
733
|
};
|
|
486
|
-
const
|
|
487
|
-
|
|
488
|
-
limitParam: "limit",
|
|
489
|
-
cursorParam: "cursor",
|
|
490
|
-
cursorPath: "next_cursor",
|
|
491
|
-
itemsPath: "data"
|
|
734
|
+
const setUserVersion = (db, version) => {
|
|
735
|
+
db.exec(`PRAGMA user_version = ${version}`);
|
|
492
736
|
};
|
|
493
|
-
const
|
|
494
|
-
|
|
495
|
-
|
|
496
|
-
|
|
497
|
-
|
|
498
|
-
|
|
737
|
+
const migrate = (db) => {
|
|
738
|
+
const currentVersion = readUserVersion(db);
|
|
739
|
+
if (currentVersion >= MIGRATIONS.length) return {
|
|
740
|
+
fromVersion: currentVersion,
|
|
741
|
+
toVersion: currentVersion,
|
|
742
|
+
applied: 0
|
|
743
|
+
};
|
|
744
|
+
for (let index = currentVersion; index < MIGRATIONS.length; index += 1) {
|
|
745
|
+
const migration = MIGRATIONS[index];
|
|
746
|
+
if (!migration) throw new Error(`Missing migration at index ${index}`);
|
|
747
|
+
db.transaction(() => {
|
|
748
|
+
db.exec(migration.sql);
|
|
749
|
+
setUserVersion(db, index + 1);
|
|
750
|
+
});
|
|
751
|
+
}
|
|
752
|
+
return {
|
|
753
|
+
fromVersion: currentVersion,
|
|
754
|
+
toVersion: MIGRATIONS.length,
|
|
755
|
+
applied: MIGRATIONS.length - currentVersion
|
|
756
|
+
};
|
|
499
757
|
};
|
|
500
|
-
const
|
|
501
|
-
const
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
505
|
-
|
|
506
|
-
|
|
758
|
+
const assertAtLatest = (db) => {
|
|
759
|
+
const currentVersion = readUserVersion(db);
|
|
760
|
+
if (currentVersion < MIGRATIONS.length) throw new Error(`Local SQLite cache schema is at version ${currentVersion}; expected ${MIGRATIONS.length}`);
|
|
761
|
+
};
|
|
762
|
+
//#endregion
|
|
763
|
+
//#region ../../packages/sync-engine/src/sqlite.ts
|
|
764
|
+
const loadModule = createRequire(import.meta.url);
|
|
765
|
+
const isBunRuntime = () => typeof Bun !== "undefined";
|
|
766
|
+
const normalizeValue = (value) => {
|
|
767
|
+
if (value === null || value === void 0) return null;
|
|
768
|
+
if (typeof value === "boolean") return value ? 1 : 0;
|
|
769
|
+
return value;
|
|
770
|
+
};
|
|
771
|
+
const normalizeParams$1 = (params) => {
|
|
772
|
+
if (!params) return;
|
|
773
|
+
return Object.fromEntries(Object.entries(params).map(([key, value]) => [key, normalizeValue(value)]));
|
|
507
774
|
};
|
|
508
|
-
const
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
const epochMs = Date.parse(value);
|
|
512
|
-
return Number.isNaN(epochMs) ? null : epochMs;
|
|
775
|
+
const invokeWithParams = (fn, params) => {
|
|
776
|
+
const normalizedParams = normalizeParams$1(params);
|
|
777
|
+
return normalizedParams ? fn(normalizedParams) : fn();
|
|
513
778
|
};
|
|
514
|
-
const
|
|
515
|
-
|
|
516
|
-
|
|
779
|
+
const wrapStatement = (statement) => ({
|
|
780
|
+
run: (params) => invokeWithParams(statement.run.bind(statement), params),
|
|
781
|
+
get: (params) => invokeWithParams(statement.get.bind(statement), params),
|
|
782
|
+
all: (params) => invokeWithParams(statement.all.bind(statement), params)
|
|
783
|
+
});
|
|
784
|
+
const openRuntimeDatabase = (path, opts) => {
|
|
785
|
+
if (isBunRuntime()) {
|
|
786
|
+
const { Database } = loadModule("bun:sqlite");
|
|
787
|
+
return new Database(path, {
|
|
788
|
+
readonly: opts?.readonly,
|
|
789
|
+
strict: true
|
|
790
|
+
});
|
|
791
|
+
}
|
|
792
|
+
const { DatabaseSync } = loadModule("node:sqlite");
|
|
793
|
+
return opts?.readonly ? new DatabaseSync(path, { readOnly: true }) : new DatabaseSync(path);
|
|
517
794
|
};
|
|
518
|
-
const
|
|
519
|
-
const
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
created_at: isoToEpochMs$1(item.created_at),
|
|
523
|
-
object: read$1(item.object),
|
|
524
|
-
url: read$1(item.url),
|
|
525
|
-
user_id: read$1(user.id),
|
|
526
|
-
user_object: read$1(user.object),
|
|
527
|
-
user_type: read$1(user.type),
|
|
528
|
-
user_display_name: read$1(user.display_name),
|
|
529
|
-
user_public_identifier: read$1(user.public_identifier),
|
|
530
|
-
user_profile_url: read$1(user.profile_url),
|
|
531
|
-
user_public_picture_url: read$1(user.public_picture_url),
|
|
532
|
-
user_description: read$1(user.description),
|
|
533
|
-
user_headline: userHeadline(user),
|
|
534
|
-
raw: JSON.stringify(item)
|
|
795
|
+
const openDatabase = (path, opts) => {
|
|
796
|
+
const database = openRuntimeDatabase(path, opts);
|
|
797
|
+
const exec = (sql) => {
|
|
798
|
+
database.exec(sql);
|
|
535
799
|
};
|
|
536
|
-
};
|
|
537
|
-
const liftConversation = (item) => {
|
|
538
|
-
const user = asRecord$1(item.user);
|
|
539
|
-
const userSpecifics = asRecord$1(user.specifics);
|
|
540
800
|
return {
|
|
541
|
-
|
|
542
|
-
|
|
543
|
-
|
|
544
|
-
|
|
545
|
-
|
|
546
|
-
|
|
547
|
-
|
|
548
|
-
|
|
549
|
-
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
|
|
556
|
-
is_group: toBool$1(item.is_group),
|
|
557
|
-
is_pinned: toBool$1(item.is_pinned),
|
|
558
|
-
is_readonly: toBool$1(item.is_readonly),
|
|
559
|
-
unread_count: read$1(item.unread_count),
|
|
560
|
-
user_object: read$1(user.object),
|
|
561
|
-
user_type: read$1(user.type),
|
|
562
|
-
user_display_name: read$1(user.display_name),
|
|
563
|
-
user_public_identifier: read$1(user.public_identifier),
|
|
564
|
-
user_profile_url: read$1(user.profile_url),
|
|
565
|
-
user_public_picture_url: read$1(user.public_picture_url),
|
|
566
|
-
user_description: read$1(user.description),
|
|
567
|
-
user_specifics_network_distance: read$1(userSpecifics.network_distance),
|
|
568
|
-
raw: JSON.stringify(item)
|
|
801
|
+
exec,
|
|
802
|
+
prepare: (sql) => wrapStatement(database.prepare(sql)),
|
|
803
|
+
transaction: (fn) => {
|
|
804
|
+
exec("BEGIN");
|
|
805
|
+
try {
|
|
806
|
+
fn();
|
|
807
|
+
exec("COMMIT");
|
|
808
|
+
} catch (error) {
|
|
809
|
+
exec("ROLLBACK");
|
|
810
|
+
throw error;
|
|
811
|
+
}
|
|
812
|
+
},
|
|
813
|
+
close: () => {
|
|
814
|
+
database.close();
|
|
815
|
+
}
|
|
569
816
|
};
|
|
570
817
|
};
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
text: read$1(item.text),
|
|
581
|
-
sender_id: read$1(item.sender_id),
|
|
582
|
-
is_deleted: toBool$1(item.is_deleted),
|
|
583
|
-
is_delivered: toBool$1(item.is_delivered),
|
|
584
|
-
is_edited: toBool$1(item.is_edited),
|
|
585
|
-
is_event: toBool$1(item.is_event),
|
|
586
|
-
is_hidden: toBool$1(item.is_hidden),
|
|
587
|
-
is_mentionned: toBool$1(item.is_mentionned),
|
|
588
|
-
is_pinned: toBool$1(item.is_pinned),
|
|
589
|
-
is_seen: toBool$1(item.is_seen),
|
|
590
|
-
is_sender: toBool$1(item.is_sender),
|
|
591
|
-
sender_object: read$1(sender.object),
|
|
592
|
-
sender_type: read$1(sender.type),
|
|
593
|
-
sender_display_name: read$1(sender.display_name),
|
|
594
|
-
sender_public_identifier: read$1(sender.public_identifier),
|
|
595
|
-
sender_profile_url: read$1(sender.profile_url),
|
|
596
|
-
sender_public_picture_url: read$1(sender.public_picture_url),
|
|
597
|
-
sender_description: read$1(sender.description),
|
|
598
|
-
sender_specifics_network_distance: read$1(senderSpecifics.network_distance),
|
|
599
|
-
raw: JSON.stringify(item)
|
|
600
|
-
};
|
|
818
|
+
//#endregion
|
|
819
|
+
//#region ../../packages/sync-engine/src/cache.ts
|
|
820
|
+
var NeverSyncedError = class extends Error {
|
|
821
|
+
name = "NeverSyncedError";
|
|
822
|
+
tables;
|
|
823
|
+
constructor(tables, message) {
|
|
824
|
+
super(message);
|
|
825
|
+
this.tables = tables;
|
|
826
|
+
}
|
|
601
827
|
};
|
|
602
|
-
const
|
|
603
|
-
|
|
604
|
-
|
|
605
|
-
|
|
606
|
-
|
|
607
|
-
|
|
608
|
-
|
|
609
|
-
|
|
610
|
-
|
|
611
|
-
|
|
612
|
-
|
|
613
|
-
|
|
614
|
-
|
|
615
|
-
|
|
616
|
-
|
|
617
|
-
|
|
618
|
-
|
|
619
|
-
|
|
620
|
-
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
analytics_impressions_counter: read$1(analytics.impressions_counter),
|
|
625
|
-
author_id: read$1(author.id),
|
|
626
|
-
author_object: read$1(author.object),
|
|
627
|
-
author_type: read$1(author.type),
|
|
628
|
-
author_display_name: read$1(author.display_name),
|
|
629
|
-
author_public_identifier: read$1(author.public_identifier),
|
|
630
|
-
author_profile_url: read$1(author.profile_url),
|
|
631
|
-
author_public_picture_url: read$1(author.public_picture_url),
|
|
632
|
-
author_description: read$1(author.description),
|
|
633
|
-
author_specifics_network_distance: read$1(authorSpecifics.network_distance),
|
|
634
|
-
permissions_can_post_comments: toBool$1(permissions.can_post_comments),
|
|
635
|
-
permissions_can_react: toBool$1(permissions.can_react),
|
|
636
|
-
permissions_can_share: toBool$1(permissions.can_share),
|
|
637
|
-
poll_id: read$1(poll.id),
|
|
638
|
-
poll_question: read$1(poll.question),
|
|
639
|
-
poll_is_open: toBool$1(poll.is_open),
|
|
640
|
-
poll_total_votes_count: read$1(poll.total_votes_count),
|
|
641
|
-
quoted_post_id: read$1(quotedPost.id),
|
|
642
|
-
quoted_post_object: read$1(quotedPost.object),
|
|
643
|
-
quoted_post_created_at: isoToEpochMs$1(quotedPost.created_at),
|
|
644
|
-
quoted_post_text: read$1(quotedPost.text),
|
|
645
|
-
quoted_post_share_url: read$1(quotedPost.share_url),
|
|
646
|
-
quoted_post_user_reacted: readReaction(quotedPost.user_reacted),
|
|
647
|
-
quoted_post_comments_counter: read$1(quotedPost.comments_counter),
|
|
648
|
-
quoted_post_reposts_counter: read$1(quotedPost.reposts_counter),
|
|
649
|
-
quoted_post_permissions_can_post_comments: toBool$1(quotedPostPermissions.can_post_comments),
|
|
650
|
-
quoted_post_permissions_can_react: toBool$1(quotedPostPermissions.can_react),
|
|
651
|
-
quoted_post_permissions_can_share: toBool$1(quotedPostPermissions.can_share),
|
|
652
|
-
quoted_post_author_id: read$1(quotedPostAuthor.id),
|
|
653
|
-
quoted_post_author_object: read$1(quotedPostAuthor.object),
|
|
654
|
-
quoted_post_author_type: read$1(quotedPostAuthor.type),
|
|
655
|
-
quoted_post_author_display_name: read$1(quotedPostAuthor.display_name),
|
|
656
|
-
quoted_post_author_public_identifier: read$1(quotedPostAuthor.public_identifier),
|
|
657
|
-
quoted_post_author_profile_url: read$1(quotedPostAuthor.profile_url),
|
|
658
|
-
quoted_post_author_public_picture_url: read$1(quotedPostAuthor.public_picture_url),
|
|
659
|
-
quoted_post_author_description: read$1(quotedPostAuthor.description),
|
|
660
|
-
quoted_post_author_specifics_network_distance: read$1(quotedPostAuthorSpecifics.network_distance),
|
|
661
|
-
reposted_by_id: read$1(repostedBy.id),
|
|
662
|
-
reposted_by_object: read$1(repostedBy.object),
|
|
663
|
-
reposted_by_type: read$1(repostedBy.type),
|
|
664
|
-
reposted_by_display_name: read$1(repostedBy.display_name),
|
|
665
|
-
reposted_by_public_identifier: read$1(repostedBy.public_identifier),
|
|
666
|
-
reposted_by_profile_url: read$1(repostedBy.profile_url),
|
|
667
|
-
reposted_by_public_picture_url: read$1(repostedBy.public_picture_url),
|
|
668
|
-
raw: JSON.stringify(item)
|
|
669
|
-
};
|
|
828
|
+
const publicCacheTables = [
|
|
829
|
+
"x_tweets",
|
|
830
|
+
"x_followers",
|
|
831
|
+
"x_following",
|
|
832
|
+
"x_bookmarks",
|
|
833
|
+
"x_profiles",
|
|
834
|
+
"x_messages",
|
|
835
|
+
"li_conversations",
|
|
836
|
+
"li_messages",
|
|
837
|
+
"li_connections",
|
|
838
|
+
"li_posts",
|
|
839
|
+
"li_profiles",
|
|
840
|
+
"li_requests",
|
|
841
|
+
"sync_state"
|
|
842
|
+
];
|
|
843
|
+
const rawCacheTables = publicCacheTables.filter((table) => table !== "sync_state").map((table) => `${table}_raw`);
|
|
844
|
+
const cacheTables = new Set([...publicCacheTables, ...rawCacheTables]);
|
|
845
|
+
const profileTables = new Set(["x_profiles_raw", "li_profiles_raw"]);
|
|
846
|
+
const sharedTables = new Set(["sync_state"]);
|
|
847
|
+
const platformTables = {
|
|
848
|
+
linkedin: new Set([...cacheTables].filter((table) => table.startsWith("li_"))),
|
|
849
|
+
x: new Set([...cacheTables].filter((table) => table.startsWith("x_")))
|
|
670
850
|
};
|
|
671
|
-
const
|
|
672
|
-
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
851
|
+
const sanitizeAccountId = (accountId) => accountId.replaceAll(/[^A-Za-z0-9._-]/g, "_");
|
|
852
|
+
const quoteIdentifier = (identifier) => `\`${identifier.replaceAll("`", "``")}\``;
|
|
853
|
+
const resolveCachePath = (path) => path === ":memory:" ? path : resolve(path);
|
|
854
|
+
const rawTableFor = (table) => table === "sync_state" ? table : `${table}_raw`;
|
|
855
|
+
const collectionForReadTable = (table) => table.endsWith("_raw") ? table.slice(0, -4) : table;
|
|
856
|
+
const physicalTableForCacheTable = (table) => table === "sync_state" || table.endsWith("_raw") ? table : rawTableFor(table);
|
|
857
|
+
const stringOrNull = (value) => typeof value === "string" ? value : null;
|
|
858
|
+
const numberOrNull = (value) => typeof value === "number" ? value : null;
|
|
859
|
+
const assertKnownTable = (table) => {
|
|
860
|
+
if (!cacheTables.has(table)) throw new Error(`Unknown cache table: ${table}`);
|
|
861
|
+
};
|
|
862
|
+
const tablesForPlatform = (platform) => new Set([...platformTables[platform], ...sharedTables]);
|
|
863
|
+
const knownTableTokens = (sql) => {
|
|
864
|
+
const tokens = sql.match(/\b[A-Za-z_][A-Za-z0-9_]*\b/g) ?? [];
|
|
865
|
+
const tables = [];
|
|
866
|
+
const seen = /* @__PURE__ */ new Set();
|
|
867
|
+
for (const token of tokens) {
|
|
868
|
+
const table = token.toLowerCase();
|
|
869
|
+
if (cacheTables.has(table) && !seen.has(table)) {
|
|
870
|
+
seen.add(table);
|
|
871
|
+
tables.push(table);
|
|
872
|
+
}
|
|
873
|
+
}
|
|
874
|
+
return tables;
|
|
689
875
|
};
|
|
690
|
-
const
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
876
|
+
const assertPlatformQuery = (sql, platform) => {
|
|
877
|
+
const allowedTables = tablesForPlatform(platform);
|
|
878
|
+
const rejectedTables = new Set(knownTableTokens(sql).filter((table) => !allowedTables.has(table)));
|
|
879
|
+
if (rejectedTables.size > 0) throw new Error(`Cache SQL for ${platform} cannot read ${[...rejectedTables].sort().join(", ")}.`);
|
|
880
|
+
};
|
|
881
|
+
const referencedTablesForRead = (sql, platform) => {
|
|
882
|
+
const allowedTables = platformTables[platform];
|
|
883
|
+
return knownTableTokens(sql).filter((table) => allowedTables.has(table));
|
|
884
|
+
};
|
|
885
|
+
const collectionShortName = (collection) => collection.replace(/^(x|li)_/, "");
|
|
886
|
+
const tableFreshnessPolicies = {
|
|
887
|
+
x_messages_raw: {
|
|
888
|
+
kind: "collection",
|
|
889
|
+
collection: "x_messages"
|
|
699
890
|
},
|
|
700
|
-
{
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
parentCollection: "li_conversations",
|
|
705
|
-
parentIdToken: ":chat_id",
|
|
706
|
-
pagination: linkedinCursorPagination,
|
|
707
|
-
pageSize: 250,
|
|
708
|
-
sinceField: "timestamp",
|
|
709
|
-
lift: liftMessage
|
|
891
|
+
li_conversations: {
|
|
892
|
+
kind: "collection",
|
|
893
|
+
collection: "li_conversations",
|
|
894
|
+
commandCollection: "li_messages"
|
|
710
895
|
},
|
|
711
|
-
{
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
requiresFullInitialSync: true,
|
|
716
|
-
pagination: linkedinRelationsCursorPagination,
|
|
717
|
-
pageSize: 100,
|
|
718
|
-
sinceField: "created_at",
|
|
719
|
-
lift: liftConnection
|
|
896
|
+
li_conversations_raw: {
|
|
897
|
+
kind: "collection",
|
|
898
|
+
collection: "li_conversations",
|
|
899
|
+
commandCollection: "li_messages"
|
|
720
900
|
},
|
|
721
|
-
{
|
|
722
|
-
|
|
723
|
-
platform: "linkedin"
|
|
724
|
-
requests: [{ path: "/users/:identifier/posts" }],
|
|
725
|
-
ownIdToken: ":identifier",
|
|
726
|
-
pagination: linkedinOffsetPagination,
|
|
727
|
-
pageSize: 50,
|
|
728
|
-
sinceField: "created_at",
|
|
729
|
-
lift: liftPost
|
|
901
|
+
li_profiles: {
|
|
902
|
+
kind: "platform",
|
|
903
|
+
platform: "linkedin"
|
|
730
904
|
},
|
|
731
|
-
{
|
|
732
|
-
|
|
733
|
-
platform: "linkedin"
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
pageSize: 100,
|
|
743
|
-
lift: liftRequest
|
|
905
|
+
li_profiles_raw: {
|
|
906
|
+
kind: "platform",
|
|
907
|
+
platform: "linkedin"
|
|
908
|
+
},
|
|
909
|
+
x_profiles: {
|
|
910
|
+
kind: "platform",
|
|
911
|
+
platform: "x"
|
|
912
|
+
},
|
|
913
|
+
x_profiles_raw: {
|
|
914
|
+
kind: "platform",
|
|
915
|
+
platform: "x"
|
|
744
916
|
}
|
|
745
|
-
];
|
|
746
|
-
//#endregion
|
|
747
|
-
//#region ../../packages/sync-engine/src/collections/x.ts
|
|
748
|
-
const xPagination = {
|
|
749
|
-
style: "cursor",
|
|
750
|
-
limitParam: "max_results",
|
|
751
|
-
cursorParam: "pagination_token",
|
|
752
|
-
cursorPath: "meta.next_token",
|
|
753
|
-
itemsPath: "data"
|
|
754
917
|
};
|
|
755
|
-
const
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
const asRecord = (value) => value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
759
|
-
const read = (value) => value === null || value === void 0 ? null : value;
|
|
760
|
-
const isoToEpochMs = (value) => {
|
|
761
|
-
if (typeof value === "number") return Number.isFinite(value) ? value : null;
|
|
762
|
-
if (typeof value !== "string") return null;
|
|
763
|
-
const epochMs = Date.parse(value);
|
|
764
|
-
return Number.isNaN(epochMs) ? null : epochMs;
|
|
918
|
+
const freshnessPolicyForTable = (table) => tableFreshnessPolicies[table] ?? {
|
|
919
|
+
kind: "collection",
|
|
920
|
+
collection: collectionForReadTable(table)
|
|
765
921
|
};
|
|
766
|
-
const
|
|
767
|
-
|
|
768
|
-
|
|
922
|
+
const platformCollectionPrefix = (platform) => platform === "linkedin" ? "li_" : "x_";
|
|
923
|
+
const syncCommandForPolicy = (platform, policy) => {
|
|
924
|
+
if (policy.kind === "platform") return `social ${policy.platform} sync`;
|
|
925
|
+
return `social ${platform} sync ${collectionShortName(policy.commandCollection ?? policy.collection)}`;
|
|
769
926
|
};
|
|
770
|
-
const
|
|
771
|
-
const
|
|
772
|
-
|
|
773
|
-
return {
|
|
774
|
-
|
|
775
|
-
created_at: isoToEpochMs(item.created_at),
|
|
776
|
-
author_id: read(item.author_id),
|
|
777
|
-
conversation_id: read(item.conversation_id),
|
|
778
|
-
in_reply_to_user_id: read(item.in_reply_to_user_id),
|
|
779
|
-
lang: read(item.lang),
|
|
780
|
-
reply_settings: read(item.reply_settings),
|
|
781
|
-
text: read(item.text),
|
|
782
|
-
url: read(item.url),
|
|
783
|
-
paid_partnership: toBool(item.paid_partnership),
|
|
784
|
-
possibly_sensitive: toBool(item.possibly_sensitive),
|
|
785
|
-
like_count: read(publicMetrics.like_count),
|
|
786
|
-
retweet_count: read(publicMetrics.retweet_count),
|
|
787
|
-
reply_count: read(publicMetrics.reply_count),
|
|
788
|
-
quote_count: read(publicMetrics.quote_count),
|
|
789
|
-
bookmark_count: read(publicMetrics.bookmark_count),
|
|
790
|
-
impression_count: read(publicMetrics.impression_count)
|
|
791
|
-
};
|
|
927
|
+
const missingNameForPolicy = (table, policy) => policy.kind === "platform" ? table : policy.collection;
|
|
928
|
+
const formatList = (items) => {
|
|
929
|
+
if (items.length <= 1) return items[0] ?? "";
|
|
930
|
+
if (items.length === 2) return `${items[0]} and ${items[1]}`;
|
|
931
|
+
return `${items.slice(0, -1).join(", ")}, and ${items.at(-1)}`;
|
|
792
932
|
};
|
|
793
|
-
const
|
|
794
|
-
|
|
795
|
-
raw: JSON.stringify(item)
|
|
796
|
-
});
|
|
797
|
-
const liftBookmark = (item) => {
|
|
798
|
-
const article = asRecord(item.article);
|
|
799
|
-
const noteTweet = asRecord(item.note_tweet);
|
|
800
|
-
return {
|
|
801
|
-
...liftTweetColumns(item),
|
|
802
|
-
article_cover_media: read(article.cover_media),
|
|
803
|
-
article_plain_text: read(article.plain_text),
|
|
804
|
-
article_preview_text: read(article.preview_text),
|
|
805
|
-
article_title: read(article.title),
|
|
806
|
-
note_tweet_text: read(noteTweet.text),
|
|
807
|
-
raw: JSON.stringify(item)
|
|
808
|
-
};
|
|
933
|
+
const collectionLastSynced = (db, collection) => {
|
|
934
|
+
return numberOrNull(db.prepare("SELECT last_synced FROM sync_state WHERE collection = :collection").get({ collection })?.last_synced);
|
|
809
935
|
};
|
|
810
|
-
const
|
|
811
|
-
|
|
812
|
-
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
816
|
-
|
|
817
|
-
|
|
818
|
-
|
|
819
|
-
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
825
|
-
|
|
826
|
-
|
|
936
|
+
const platformLastSynced = (db, platform) => {
|
|
937
|
+
return numberOrNull(db.prepare("SELECT MAX(last_synced) AS last_synced FROM sync_state WHERE collection GLOB :glob").get({ glob: `${platformCollectionPrefix(platform)}*` })?.last_synced);
|
|
938
|
+
};
|
|
939
|
+
const lastSyncedForPolicy = (db, policy) => policy.kind === "platform" ? platformLastSynced(db, policy.platform) : collectionLastSynced(db, policy.collection);
|
|
940
|
+
const missingFreshnessForTables = (tables) => {
|
|
941
|
+
const missing = [];
|
|
942
|
+
const seen = /* @__PURE__ */ new Set();
|
|
943
|
+
for (const table of tables) {
|
|
944
|
+
if (table.lastSynced != null) continue;
|
|
945
|
+
const key = `${table.missingName}\0${table.command}`;
|
|
946
|
+
if (!seen.has(key)) {
|
|
947
|
+
seen.add(key);
|
|
948
|
+
missing.push({
|
|
949
|
+
name: table.missingName,
|
|
950
|
+
command: table.command
|
|
951
|
+
});
|
|
952
|
+
}
|
|
953
|
+
}
|
|
954
|
+
return missing;
|
|
955
|
+
};
|
|
956
|
+
const neverSyncedMessage = (missing) => {
|
|
957
|
+
const names = missing.map((item) => item.name);
|
|
958
|
+
const commands = missing.map((item) => `\`${item.command}\``);
|
|
959
|
+
return `No synced ${formatList(names)} yet — run ${formatList(commands)} first.`;
|
|
960
|
+
};
|
|
961
|
+
const assertKnownProfileTable = (table) => {
|
|
962
|
+
assertKnownTable(table);
|
|
963
|
+
if (!profileTables.has(physicalTableForCacheTable(table))) throw new Error(`Unknown profile table: ${table}`);
|
|
964
|
+
};
|
|
965
|
+
const cacheDescriptionNotes = ["time columns (*_at, muted_until, synced_at) are epoch ms; booleans are 0/1", "each view has a <name>_raw twin holding the upstream JSON: json_extract(raw,'$.field')"];
|
|
966
|
+
const metadataForPlatform = (platform) => {
|
|
967
|
+
const platforms = platform ? [platform] : ["linkedin", "x"];
|
|
827
968
|
return {
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
name: read(item.name),
|
|
832
|
-
description: read(item.description),
|
|
833
|
-
location: read(item.location),
|
|
834
|
-
url: read(item.url),
|
|
835
|
-
is_identity_verified: toBool(item.is_identity_verified),
|
|
836
|
-
protected: toBool(item.protected),
|
|
837
|
-
verified: toBool(item.verified),
|
|
838
|
-
verified_type: read(item.verified_type),
|
|
839
|
-
subscription_type: read(item.subscription_type),
|
|
840
|
-
parody: toBool(item.parody),
|
|
841
|
-
most_recent_tweet_id: read(item.most_recent_tweet_id),
|
|
842
|
-
pinned_tweet_id: read(item.pinned_tweet_id),
|
|
843
|
-
profile_banner_url: read(item.profile_banner_url),
|
|
844
|
-
profile_image_url: read(item.profile_image_url),
|
|
845
|
-
affiliation_badge_url: read(affiliation.badge_url),
|
|
846
|
-
affiliation_description: read(affiliation.description),
|
|
847
|
-
affiliation_url: read(affiliation.url),
|
|
848
|
-
subscription_subscribes_to_you: toBool(subscription.subscribes_to_you),
|
|
849
|
-
followers_count: read(publicMetrics.followers_count),
|
|
850
|
-
following_count: read(publicMetrics.following_count),
|
|
851
|
-
tweet_count: read(publicMetrics.tweet_count),
|
|
852
|
-
listed_count: read(publicMetrics.listed_count),
|
|
853
|
-
like_count: read(publicMetrics.like_count),
|
|
854
|
-
media_count: read(publicMetrics.media_count),
|
|
855
|
-
raw: JSON.stringify(item)
|
|
969
|
+
joins: platforms.flatMap((item) => CACHE_METADATA_BY_PLATFORM[item].joins),
|
|
970
|
+
enums: Object.assign({}, ...platforms.map((item) => CACHE_METADATA_BY_PLATFORM[item].enums)),
|
|
971
|
+
indexed: Object.assign({}, ...platforms.map((item) => CACHE_METADATA_BY_PLATFORM[item].indexed))
|
|
856
972
|
};
|
|
857
973
|
};
|
|
858
|
-
|
|
859
|
-
|
|
860
|
-
const
|
|
861
|
-
{
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
974
|
+
const cachePath = (args) => join(homedir(), ".social", "cache", `${args.platform}-${sanitizeAccountId(args.accountId)}.db`);
|
|
975
|
+
const openCacheAt = (path, opts) => {
|
|
976
|
+
const dbPath = resolveCachePath(path);
|
|
977
|
+
if (opts?.readonly) {
|
|
978
|
+
const db = openDatabase(dbPath, { readonly: true });
|
|
979
|
+
assertAtLatest(db);
|
|
980
|
+
return createCache(dbPath, db);
|
|
981
|
+
}
|
|
982
|
+
if (dbPath !== ":memory:") mkdirSync(dirname(dbPath), { recursive: true });
|
|
983
|
+
const db = openDatabase(dbPath);
|
|
984
|
+
const result = migrate(db);
|
|
985
|
+
if (result.applied > 0) opts?.onMigrate?.(result);
|
|
986
|
+
return createCache(dbPath, db);
|
|
987
|
+
};
|
|
988
|
+
const openCache = (args) => openCacheAt(cachePath(args), {
|
|
989
|
+
readonly: args.readonly,
|
|
990
|
+
onMigrate: args.onMigrate
|
|
991
|
+
});
|
|
992
|
+
const createCache = (path, db) => ({
|
|
993
|
+
upsert: (table, rows, syncedAt = Date.now()) => {
|
|
994
|
+
if (rows.length === 0) return;
|
|
995
|
+
assertKnownTable(table);
|
|
996
|
+
const physicalTable = physicalTableForCacheTable(table);
|
|
997
|
+
const columns = [...Object.keys(rows[0] ?? {}).filter((column) => column !== "synced_at"), "synced_at"];
|
|
998
|
+
const updateColumns = columns.filter((column) => column !== "id");
|
|
999
|
+
const statement = db.prepare(`INSERT INTO ${quoteIdentifier(physicalTable)} (${columns.map(quoteIdentifier).join(", ")}) VALUES (${columns.map((column) => `:${column}`).join(", ")}) ON CONFLICT(id) DO UPDATE SET ${updateColumns.map((column) => `${quoteIdentifier(column)}=excluded.${quoteIdentifier(column)}`).join(", ")}`);
|
|
1000
|
+
db.transaction(() => {
|
|
1001
|
+
for (const row of rows) statement.run({
|
|
1002
|
+
...row,
|
|
1003
|
+
synced_at: syncedAt
|
|
1004
|
+
});
|
|
1005
|
+
});
|
|
870
1006
|
},
|
|
871
|
-
{
|
|
872
|
-
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
1007
|
+
upsertProfiles: (table, rows) => {
|
|
1008
|
+
if (rows.length === 0) return;
|
|
1009
|
+
assertKnownProfileTable(table);
|
|
1010
|
+
const physicalTable = physicalTableForCacheTable(table);
|
|
1011
|
+
const rowsByProviderId = /* @__PURE__ */ new Map();
|
|
1012
|
+
for (const row of rows) rowsByProviderId.set(row.providerId, row);
|
|
1013
|
+
const columns = [
|
|
1014
|
+
"provider_id",
|
|
1015
|
+
"username",
|
|
1016
|
+
"display_name",
|
|
1017
|
+
"avatar_url",
|
|
1018
|
+
"headline",
|
|
1019
|
+
"raw",
|
|
1020
|
+
"synced_at"
|
|
1021
|
+
];
|
|
1022
|
+
const quotedTable = quoteIdentifier(physicalTable);
|
|
1023
|
+
const nullableColumns = [
|
|
1024
|
+
"username",
|
|
1025
|
+
"avatar_url",
|
|
1026
|
+
"headline"
|
|
1027
|
+
];
|
|
1028
|
+
const lastWriteColumns = [
|
|
1029
|
+
"display_name",
|
|
1030
|
+
"raw",
|
|
1031
|
+
"synced_at"
|
|
1032
|
+
];
|
|
1033
|
+
const updateAssignments = [...nullableColumns.map((column) => `${quoteIdentifier(column)}=COALESCE(excluded.${quoteIdentifier(column)}, ${quotedTable}.${quoteIdentifier(column)})`), ...lastWriteColumns.map((column) => `${quoteIdentifier(column)}=excluded.${quoteIdentifier(column)}`)].join(", ");
|
|
1034
|
+
const statement = db.prepare(`INSERT INTO ${quotedTable} (${columns.map(quoteIdentifier).join(", ")}) VALUES (${columns.map((column) => `:${column}`).join(", ")}) ON CONFLICT(provider_id) DO UPDATE SET ${updateAssignments}`);
|
|
1035
|
+
const syncedAt = Date.now();
|
|
1036
|
+
db.transaction(() => {
|
|
1037
|
+
for (const row of rowsByProviderId.values()) statement.run({
|
|
1038
|
+
provider_id: row.providerId,
|
|
1039
|
+
username: row.username,
|
|
1040
|
+
display_name: row.displayName,
|
|
1041
|
+
avatar_url: row.avatarURL,
|
|
1042
|
+
headline: row.headline,
|
|
1043
|
+
raw: JSON.stringify(row.raw),
|
|
1044
|
+
synced_at: syncedAt
|
|
1045
|
+
});
|
|
1046
|
+
});
|
|
880
1047
|
},
|
|
881
|
-
{
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
|
|
887
|
-
|
|
888
|
-
|
|
889
|
-
lift: liftXUser
|
|
1048
|
+
getCursor: (collection) => {
|
|
1049
|
+
const row = db.prepare("SELECT cursor, last_synced, object_count FROM sync_state WHERE collection = :collection").get({ collection });
|
|
1050
|
+
if (!row) return;
|
|
1051
|
+
return {
|
|
1052
|
+
cursor: stringOrNull(row.cursor),
|
|
1053
|
+
lastSynced: numberOrNull(row.last_synced),
|
|
1054
|
+
objectCount: numberOrNull(row.object_count)
|
|
1055
|
+
};
|
|
890
1056
|
},
|
|
891
|
-
{
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
895
|
-
|
|
896
|
-
|
|
897
|
-
|
|
898
|
-
lift: liftBookmark
|
|
1057
|
+
setCursor: (collection, cursor, objectCount, lastSynced = Date.now()) => {
|
|
1058
|
+
db.prepare("INSERT INTO sync_state (collection, cursor, last_synced, object_count) VALUES (:collection, :cursor, :last_synced, :object_count) ON CONFLICT(collection) DO UPDATE SET cursor=excluded.cursor, last_synced=excluded.last_synced, object_count=excluded.object_count").run({
|
|
1059
|
+
collection,
|
|
1060
|
+
cursor,
|
|
1061
|
+
last_synced: lastSynced,
|
|
1062
|
+
object_count: objectCount
|
|
1063
|
+
});
|
|
899
1064
|
},
|
|
900
|
-
{
|
|
901
|
-
|
|
902
|
-
|
|
903
|
-
|
|
904
|
-
|
|
905
|
-
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
1065
|
+
reset: ({ tables, collections }) => {
|
|
1066
|
+
for (const table of tables) assertKnownTable(table);
|
|
1067
|
+
const results = [];
|
|
1068
|
+
db.transaction(() => {
|
|
1069
|
+
for (const table of tables) {
|
|
1070
|
+
const physicalTable = physicalTableForCacheTable(table);
|
|
1071
|
+
const result = db.prepare(`DELETE FROM ${quoteIdentifier(physicalTable)}`).run();
|
|
1072
|
+
results.push({
|
|
1073
|
+
table: physicalTable,
|
|
1074
|
+
rowsDeleted: result?.changes ?? 0
|
|
1075
|
+
});
|
|
910
1076
|
}
|
|
911
|
-
|
|
912
|
-
|
|
913
|
-
|
|
914
|
-
|
|
915
|
-
|
|
1077
|
+
const deleteState = db.prepare("DELETE FROM sync_state WHERE collection = :collection");
|
|
1078
|
+
for (const collection of collections) deleteState.run({ collection });
|
|
1079
|
+
});
|
|
1080
|
+
return results;
|
|
1081
|
+
},
|
|
1082
|
+
query: (sql, opts) => {
|
|
1083
|
+
if (opts?.platform) assertPlatformQuery(sql, opts.platform);
|
|
1084
|
+
return db.prepare(sql).all();
|
|
1085
|
+
},
|
|
1086
|
+
read: (sql, opts) => {
|
|
1087
|
+
assertPlatformQuery(sql, opts.platform);
|
|
1088
|
+
const tables = referencedTablesForRead(sql, opts.platform).map((name) => {
|
|
1089
|
+
const policy = freshnessPolicyForTable(name);
|
|
1090
|
+
return {
|
|
1091
|
+
name,
|
|
1092
|
+
missingName: missingNameForPolicy(name, policy),
|
|
1093
|
+
command: syncCommandForPolicy(opts.platform, policy),
|
|
1094
|
+
lastSynced: lastSyncedForPolicy(db, policy)
|
|
1095
|
+
};
|
|
1096
|
+
});
|
|
1097
|
+
const missingFreshness = missingFreshnessForTables(tables);
|
|
1098
|
+
if (missingFreshness.length > 0) throw new NeverSyncedError(missingFreshness.map((item) => item.name), neverSyncedMessage(missingFreshness));
|
|
1099
|
+
return {
|
|
1100
|
+
rows: db.prepare(sql).all(),
|
|
1101
|
+
tables: tables.map(({ name, lastSynced }) => ({
|
|
1102
|
+
name,
|
|
1103
|
+
lastSynced
|
|
1104
|
+
}))
|
|
1105
|
+
};
|
|
1106
|
+
},
|
|
1107
|
+
describe: (opts) => {
|
|
1108
|
+
const allowedTables = opts?.platform ? tablesForPlatform(opts.platform) : void 0;
|
|
1109
|
+
const metadata = metadataForPlatform(opts?.platform);
|
|
1110
|
+
const tables = db.prepare("SELECT name FROM sqlite_master WHERE type IN ('table', 'view') AND name NOT LIKE 'sqlite_%' ORDER BY name").all().filter((table) => !String(table.name).endsWith("_raw")).filter((table) => allowedTables === void 0 || allowedTables.has(String(table.name)));
|
|
1111
|
+
const lastSyncedForTable = (name) => {
|
|
1112
|
+
if (name === "sync_state") return null;
|
|
1113
|
+
return lastSyncedForPolicy(db, freshnessPolicyForTable(name));
|
|
1114
|
+
};
|
|
1115
|
+
const columnsForTable = (name) => db.prepare(`PRAGMA table_info(${quoteIdentifier(name)})`).all().map((row) => String(row.name)).join(", ");
|
|
1116
|
+
return {
|
|
1117
|
+
path,
|
|
1118
|
+
notes: cacheDescriptionNotes,
|
|
1119
|
+
joins: metadata.joins,
|
|
1120
|
+
enums: metadata.enums,
|
|
1121
|
+
tables: tables.map((table) => {
|
|
1122
|
+
const name = String(table.name);
|
|
1123
|
+
const rowCount = db.prepare(`SELECT count(*) AS rows FROM ${quoteIdentifier(name)}`).get();
|
|
1124
|
+
const syncedAt = lastSyncedForTable(name);
|
|
1125
|
+
return {
|
|
1126
|
+
name,
|
|
1127
|
+
rows: Number(rowCount?.rows ?? 0),
|
|
1128
|
+
synced_at: syncedAt,
|
|
1129
|
+
query_ready: syncedAt != null,
|
|
1130
|
+
columns: columnsForTable(name),
|
|
1131
|
+
indexed: metadata.indexed[name] ?? []
|
|
1132
|
+
};
|
|
1133
|
+
})
|
|
1134
|
+
};
|
|
1135
|
+
},
|
|
1136
|
+
close: () => {
|
|
1137
|
+
db.close();
|
|
916
1138
|
}
|
|
917
|
-
|
|
918
|
-
const collectionByKey = (key) => COLLECTIONS.find((collection) => collection.key === key);
|
|
919
|
-
const collectionsFor = (platform) => COLLECTIONS.filter((collection) => collection.platform === platform);
|
|
920
|
-
const syncableCollectionsFor = (platform) => {
|
|
921
|
-
const parentKeys = new Set(COLLECTIONS.map((collection) => collection.parentCollection).filter(Boolean));
|
|
922
|
-
return collectionsFor(platform).filter((collection) => !parentKeys.has(collection.key));
|
|
923
|
-
};
|
|
924
|
-
const physicalTableFor = (collection) => collection.table ?? collection.key;
|
|
925
|
-
const resetTablesFor = (collection) => {
|
|
926
|
-
if (!collection.parentCollection) return [collection];
|
|
927
|
-
const parent = collectionByKey(collection.parentCollection);
|
|
928
|
-
return parent ? [collection, parent] : [collection];
|
|
929
|
-
};
|
|
930
|
-
const resetPhysicalTablesFor = (collection) => resetTablesFor(collection).map(physicalTableFor);
|
|
1139
|
+
});
|
|
931
1140
|
//#endregion
|
|
932
1141
|
//#region ../../packages/sync-engine/src/cost.ts
|
|
933
1142
|
const creditsFromHeader = (headers) => {
|
|
@@ -937,7 +1146,7 @@ const creditsFromHeader = (headers) => {
|
|
|
937
1146
|
};
|
|
938
1147
|
//#endregion
|
|
939
1148
|
//#region ../../packages/sync-engine/src/profiles.ts
|
|
940
|
-
const profileTableFor = (platform) => platform === "x" ? "
|
|
1149
|
+
const profileTableFor = (platform) => platform === "x" ? "x_profiles_raw" : "li_profiles_raw";
|
|
941
1150
|
//#endregion
|
|
942
1151
|
//#region ../../node_modules/.bun/drizzle-orm@0.43.1+ae5d55cf414a93be/node_modules/drizzle-orm/entity.js
|
|
943
1152
|
const entityKind = Symbol.for("drizzle:entityKind");
|
|
@@ -3698,7 +3907,7 @@ const rawColumns = () => ({
|
|
|
3698
3907
|
raw: text$1("raw").notNull(),
|
|
3699
3908
|
syncedAt: integer$2("synced_at").notNull()
|
|
3700
3909
|
});
|
|
3701
|
-
sqliteTable("
|
|
3910
|
+
sqliteTable("x_tweets_raw", {
|
|
3702
3911
|
id: idColumn(),
|
|
3703
3912
|
createdAt: integer$2("created_at"),
|
|
3704
3913
|
authorId: text$1("author_id"),
|
|
@@ -3722,7 +3931,7 @@ sqliteTable("x_tweets", {
|
|
|
3722
3931
|
index$1("x_tweets_author_id_created_at_idx").on(table.authorId, table.createdAt),
|
|
3723
3932
|
index$1("x_tweets_conversation_id_created_at_idx").on(table.conversationId, table.createdAt)
|
|
3724
3933
|
]);
|
|
3725
|
-
sqliteTable("
|
|
3934
|
+
sqliteTable("x_bookmarks_raw", {
|
|
3726
3935
|
id: idColumn(),
|
|
3727
3936
|
createdAt: integer$2("created_at"),
|
|
3728
3937
|
authorId: text$1("author_id"),
|
|
@@ -3753,14 +3962,14 @@ sqliteTable("x_bookmarks", {
|
|
|
3753
3962
|
]);
|
|
3754
3963
|
const profileColumns = () => ({
|
|
3755
3964
|
providerId: text$1("provider_id").primaryKey(),
|
|
3756
|
-
|
|
3965
|
+
username: text$1("username"),
|
|
3757
3966
|
displayName: text$1("display_name").notNull(),
|
|
3758
3967
|
avatarURL: text$1("avatar_url"),
|
|
3759
3968
|
headline: text$1("headline"),
|
|
3760
3969
|
...rawColumns()
|
|
3761
3970
|
});
|
|
3762
|
-
sqliteTable("
|
|
3763
|
-
sqliteTable("
|
|
3971
|
+
sqliteTable("x_profiles_raw", profileColumns(), (table) => [index$1("x_profiles_username_idx").on(table.username)]);
|
|
3972
|
+
sqliteTable("x_messages_raw", {
|
|
3764
3973
|
id: idColumn(),
|
|
3765
3974
|
createdAt: integer$2("created_at"),
|
|
3766
3975
|
dmConversationId: text$1("dm_conversation_id"),
|
|
@@ -3807,21 +4016,21 @@ const xUserColumns = () => ({
|
|
|
3807
4016
|
mediaCount: integer$2("media_count"),
|
|
3808
4017
|
...rawColumns()
|
|
3809
4018
|
});
|
|
3810
|
-
sqliteTable("
|
|
4019
|
+
sqliteTable("x_followers_raw", xUserColumns(), (table) => [
|
|
3811
4020
|
index$1("x_followers_created_at_idx").on(table.createdAt),
|
|
3812
4021
|
index$1("x_followers_synced_at_idx").on(table.syncedAt),
|
|
3813
4022
|
index$1("x_followers_username_idx").on(table.username),
|
|
3814
4023
|
index$1("x_followers_verified_idx").on(table.verified),
|
|
3815
4024
|
index$1("x_followers_verified_type_idx").on(table.verifiedType)
|
|
3816
4025
|
]);
|
|
3817
|
-
sqliteTable("
|
|
4026
|
+
sqliteTable("x_following_raw", xUserColumns(), (table) => [
|
|
3818
4027
|
index$1("x_following_created_at_idx").on(table.createdAt),
|
|
3819
4028
|
index$1("x_following_synced_at_idx").on(table.syncedAt),
|
|
3820
4029
|
index$1("x_following_username_idx").on(table.username),
|
|
3821
4030
|
index$1("x_following_verified_idx").on(table.verified),
|
|
3822
4031
|
index$1("x_following_verified_type_idx").on(table.verifiedType)
|
|
3823
4032
|
]);
|
|
3824
|
-
sqliteTable("
|
|
4033
|
+
sqliteTable("li_connections_raw", {
|
|
3825
4034
|
id: idColumn(),
|
|
3826
4035
|
createdAt: integer$2("created_at"),
|
|
3827
4036
|
object: text$1("object"),
|
|
@@ -3843,8 +4052,8 @@ sqliteTable("li_connections", {
|
|
|
3843
4052
|
index$1("li_connections_user_public_identifier_idx").on(table.userPublicIdentifier),
|
|
3844
4053
|
index$1("li_connections_user_display_name_idx").on(table.userDisplayName)
|
|
3845
4054
|
]);
|
|
3846
|
-
sqliteTable("
|
|
3847
|
-
sqliteTable("
|
|
4055
|
+
sqliteTable("li_profiles_raw", profileColumns(), (table) => [index$1("li_profiles_username_idx").on(table.username)]);
|
|
4056
|
+
sqliteTable("li_conversations_raw", {
|
|
3848
4057
|
id: idColumn(),
|
|
3849
4058
|
createdAt: integer$2("created_at"),
|
|
3850
4059
|
lastMessageTimestamp: integer$2("last_message_timestamp"),
|
|
@@ -3881,7 +4090,7 @@ sqliteTable("li_conversations", {
|
|
|
3881
4090
|
index$1("li_conversations_user_display_name_idx").on(table.userDisplayName),
|
|
3882
4091
|
index$1("li_conversations_is_archived_last_message_timestamp_idx").on(table.isArchived, table.lastMessageTimestamp)
|
|
3883
4092
|
]);
|
|
3884
|
-
sqliteTable("
|
|
4093
|
+
sqliteTable("li_messages_raw", {
|
|
3885
4094
|
id: idColumn(),
|
|
3886
4095
|
timestamp: integer$2("timestamp"),
|
|
3887
4096
|
chatId: text$1("chat_id"),
|
|
@@ -3912,7 +4121,7 @@ sqliteTable("li_messages", {
|
|
|
3912
4121
|
index$1("li_messages_chat_id_timestamp_idx").on(table.chatId, table.timestamp),
|
|
3913
4122
|
index$1("li_messages_sender_id_timestamp_idx").on(table.senderId, table.timestamp)
|
|
3914
4123
|
]);
|
|
3915
|
-
sqliteTable("
|
|
4124
|
+
sqliteTable("li_posts_raw", {
|
|
3916
4125
|
id: idColumn(),
|
|
3917
4126
|
createdAt: integer$2("created_at"),
|
|
3918
4127
|
object: text$1("object"),
|
|
@@ -3973,7 +4182,7 @@ sqliteTable("li_posts", {
|
|
|
3973
4182
|
index$1("li_posts_author_id_created_at_idx").on(table.authorId, table.createdAt),
|
|
3974
4183
|
index$1("li_posts_author_public_identifier_created_at_idx").on(table.authorPublicIdentifier, table.createdAt)
|
|
3975
4184
|
]);
|
|
3976
|
-
sqliteTable("
|
|
4185
|
+
sqliteTable("li_requests_raw", {
|
|
3977
4186
|
id: idColumn(),
|
|
3978
4187
|
createdAt: integer$2("created_at"),
|
|
3979
4188
|
object: text$1("object"),
|
|
@@ -4003,19 +4212,31 @@ sqliteTable("sync_state", {
|
|
|
4003
4212
|
});
|
|
4004
4213
|
//#endregion
|
|
4005
4214
|
//#region ../../packages/sync-engine/src/since.ts
|
|
4006
|
-
const
|
|
4007
|
-
const
|
|
4008
|
-
const
|
|
4009
|
-
|
|
4010
|
-
|
|
4011
|
-
|
|
4012
|
-
|
|
4013
|
-
|
|
4215
|
+
const isoDatePattern = /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])$/;
|
|
4216
|
+
const isoDateTimePattern = /^\d{4}-(0[1-9]|1[0-2])-(0[1-9]|[12]\d|3[01])T([01]\d|2[0-3]):[0-5]\d(?::[0-5]\d(?:\.\d{1,3})?)?(?:Z|[+-]([01]\d|2[0-3]):[0-5]\d)?$/;
|
|
4217
|
+
const timezonePattern = /(Z|[+-]\d{2}:\d{2})$/;
|
|
4218
|
+
const invalidSinceMessage = "Invalid --since value. Use an ISO 8601 date like 2026-05-04 or datetime like 2026-05-04T00:00:00Z.";
|
|
4219
|
+
const daysInMonth = (year, month) => {
|
|
4220
|
+
if (month === 2) return year % 4 === 0 && (year % 100 !== 0 || year % 400 === 0) ? 29 : 28;
|
|
4221
|
+
return [
|
|
4222
|
+
4,
|
|
4223
|
+
6,
|
|
4224
|
+
9,
|
|
4225
|
+
11
|
|
4226
|
+
].includes(month) ? 30 : 31;
|
|
4227
|
+
};
|
|
4228
|
+
const hasValidDate = (value) => {
|
|
4229
|
+
const year = Number.parseInt(value.slice(0, 4), 10);
|
|
4230
|
+
const month = Number.parseInt(value.slice(5, 7), 10);
|
|
4231
|
+
return Number.parseInt(value.slice(8, 10), 10) <= daysInMonth(year, month);
|
|
4232
|
+
};
|
|
4233
|
+
const parseSince = (input) => {
|
|
4014
4234
|
const value = input.trim();
|
|
4015
|
-
const
|
|
4016
|
-
|
|
4017
|
-
const
|
|
4018
|
-
|
|
4235
|
+
const isISODate = isoDatePattern.test(value);
|
|
4236
|
+
const isISODateTime = isoDateTimePattern.test(value);
|
|
4237
|
+
const normalized = isISODateTime && !timezonePattern.test(value) ? `${value}Z` : value;
|
|
4238
|
+
const parsed = (isISODate || isISODateTime) && hasValidDate(value) ? Date.parse(normalized) : NaN;
|
|
4239
|
+
if (Number.isNaN(parsed)) throw new Error(invalidSinceMessage);
|
|
4019
4240
|
return parsed;
|
|
4020
4241
|
};
|
|
4021
4242
|
//#endregion
|
|
@@ -8672,6 +8893,7 @@ var SyncPageError = class extends Error {
|
|
|
8672
8893
|
pagesCompleted;
|
|
8673
8894
|
objectsUpserted;
|
|
8674
8895
|
creditsSpent;
|
|
8896
|
+
cursorPersisted;
|
|
8675
8897
|
cause;
|
|
8676
8898
|
constructor(context, cause) {
|
|
8677
8899
|
super(`Failed to sync ${context.collectionKey} page at ${context.path}`);
|
|
@@ -8687,13 +8909,31 @@ var SyncPageError = class extends Error {
|
|
|
8687
8909
|
this.pagesCompleted = context.pagesCompleted;
|
|
8688
8910
|
this.objectsUpserted = context.objectsUpserted;
|
|
8689
8911
|
this.creditsSpent = context.creditsSpent;
|
|
8912
|
+
this.cursorPersisted = context.cursorPersisted;
|
|
8690
8913
|
this.cause = cause;
|
|
8691
8914
|
}
|
|
8692
8915
|
};
|
|
8916
|
+
const PARTIAL_SYNC_CURSOR_PREFIX = "partial-sync-v1:";
|
|
8693
8917
|
const itemsSchema = array$1(record$1(string$1(), unknown$1()));
|
|
8694
8918
|
const cursorSchema = union$1([string$1(), _null$1()]);
|
|
8695
8919
|
const totalCountSchema = number$1().int().nonnegative();
|
|
8696
8920
|
const isRecord$5 = (value) => value !== null && typeof value === "object" && !Array.isArray(value);
|
|
8921
|
+
const validPartialSyncCursor = (value) => value.version === 1 && Number.isInteger(value.requestIndex) && typeof value.requestIndex === "number" && value.requestIndex >= 0 && typeof value.cursor === "string" && (typeof value.checkpoint === "string" || value.checkpoint === null) && (typeof value.newestId === "string" || value.newestId === null) && (typeof value.since === "number" || value.since === null);
|
|
8922
|
+
const decodePartialSyncCursor = (value) => {
|
|
8923
|
+
if (!value?.startsWith(PARTIAL_SYNC_CURSOR_PREFIX)) return;
|
|
8924
|
+
try {
|
|
8925
|
+
const parsed = JSON.parse(value.slice(16));
|
|
8926
|
+
return isRecord$5(parsed) && validPartialSyncCursor(parsed) ? parsed : void 0;
|
|
8927
|
+
} catch {
|
|
8928
|
+
return;
|
|
8929
|
+
}
|
|
8930
|
+
};
|
|
8931
|
+
const encodePartialSyncCursor = (cursor) => `${PARTIAL_SYNC_CURSOR_PREFIX}${JSON.stringify(cursor)}`;
|
|
8932
|
+
const isRateLimitError = (error) => isRecord$5(error) && error.status === 429;
|
|
8933
|
+
const retryAfterSecondsFromRateLimitError = (error) => {
|
|
8934
|
+
if (!(isRateLimitError(error) && isRecord$5(error))) return;
|
|
8935
|
+
return typeof error.retryAfterSeconds === "number" && Number.isFinite(error.retryAfterSeconds) && error.retryAfterSeconds >= 0 ? error.retryAfterSeconds : void 0;
|
|
8936
|
+
};
|
|
8697
8937
|
const getByPath = (obj, path) => {
|
|
8698
8938
|
if (path === void 0) return;
|
|
8699
8939
|
let current = obj;
|
|
@@ -8756,6 +8996,7 @@ const stepPage = (state, page, opts) => {
|
|
|
8756
8996
|
const newestId = state.firstPage && page.items[0] ? String(page.items[0].id) : state.newestId;
|
|
8757
8997
|
const rows = [];
|
|
8758
8998
|
const ids = [];
|
|
8999
|
+
let crossedSince = false;
|
|
8759
9000
|
for (const item of page.items) {
|
|
8760
9001
|
const id = String(item.id);
|
|
8761
9002
|
if (state.checkpoint !== null && id === state.checkpoint) return {
|
|
@@ -8771,17 +9012,7 @@ const stepPage = (state, page, opts) => {
|
|
|
8771
9012
|
};
|
|
8772
9013
|
if (opts.since != null && opts.collection.sinceField) {
|
|
8773
9014
|
const timestamp = liftedTimestamp(opts.collection, item);
|
|
8774
|
-
if (timestamp != null && timestamp < opts.since)
|
|
8775
|
-
rows,
|
|
8776
|
-
ids,
|
|
8777
|
-
nextState: {
|
|
8778
|
-
...state,
|
|
8779
|
-
newestId,
|
|
8780
|
-
firstPage: false
|
|
8781
|
-
},
|
|
8782
|
-
stoppedReason: "since",
|
|
8783
|
-
done: true
|
|
8784
|
-
};
|
|
9015
|
+
if (timestamp != null && timestamp < opts.since) crossedSince = true;
|
|
8785
9016
|
}
|
|
8786
9017
|
const row = opts.collection.lift(item);
|
|
8787
9018
|
applyBakedQueryOverlay(row, opts.request.bakedQuery);
|
|
@@ -8789,6 +9020,17 @@ const stepPage = (state, page, opts) => {
|
|
|
8789
9020
|
rows.push(row);
|
|
8790
9021
|
if (opts.collectIds) ids.push(id);
|
|
8791
9022
|
}
|
|
9023
|
+
if (crossedSince) return {
|
|
9024
|
+
rows,
|
|
9025
|
+
ids,
|
|
9026
|
+
nextState: {
|
|
9027
|
+
...state,
|
|
9028
|
+
newestId,
|
|
9029
|
+
firstPage: false
|
|
9030
|
+
},
|
|
9031
|
+
stoppedReason: "since",
|
|
9032
|
+
done: true
|
|
9033
|
+
};
|
|
8792
9034
|
if (opts.collection.pagination.style === "offset") {
|
|
8793
9035
|
const nextOffset = state.offset + opts.collection.pageSize;
|
|
8794
9036
|
const done = page.items.length === 0 || page.totalCount != null && nextOffset >= page.totalCount || page.totalCount == null && page.items.length < opts.collection.pageSize;
|
|
@@ -8825,18 +9067,62 @@ const mergeStoppedReason = (current, next) => {
|
|
|
8825
9067
|
return "complete";
|
|
8826
9068
|
};
|
|
8827
9069
|
const walkRequest = async (args) => {
|
|
8828
|
-
const
|
|
9070
|
+
const storedCursor = args.deps.cache.getCursor(args.collection.key)?.cursor ?? null;
|
|
9071
|
+
const storedPartialCursor = decodePartialSyncCursor(storedCursor);
|
|
9072
|
+
const checkpoint = args.resumeCursor?.checkpoint ?? (args.useCheckpoint ? storedPartialCursor?.checkpoint ?? storedCursor : null);
|
|
8829
9073
|
const ids = [];
|
|
8830
9074
|
let state = {
|
|
8831
9075
|
checkpoint,
|
|
8832
|
-
cursor: null,
|
|
9076
|
+
cursor: args.resumeCursor?.cursor ?? null,
|
|
8833
9077
|
offset: 0,
|
|
8834
|
-
newestId: null,
|
|
8835
|
-
firstPage:
|
|
9078
|
+
newestId: args.resumeCursor?.newestId ?? null,
|
|
9079
|
+
firstPage: args.resumeCursor === void 0
|
|
8836
9080
|
};
|
|
8837
9081
|
let pages = 0;
|
|
8838
9082
|
let upserted = 0;
|
|
8839
9083
|
let stoppedReason = "complete";
|
|
9084
|
+
const rateLimitNow = args.rateLimit?.now ?? Date.now;
|
|
9085
|
+
const rateLimitSleep = args.rateLimit?.sleep ?? sleep;
|
|
9086
|
+
const rateLimitMaxRetries = args.rateLimit?.maxRetries ?? 25;
|
|
9087
|
+
const pageRequestOptions = {
|
|
9088
|
+
headers: args.deps.accountHeaders,
|
|
9089
|
+
...args.rateLimit?.timeoutSeconds === void 0 ? {} : { retry: { limit: 0 } }
|
|
9090
|
+
};
|
|
9091
|
+
const waitForRateLimit = async (error) => {
|
|
9092
|
+
if (!(args.rateLimit && args.rateLimitState?.deadlineMs !== void 0)) return false;
|
|
9093
|
+
const retryCount = args.rateLimitState.retryCount + 1;
|
|
9094
|
+
const retryAfterSeconds = args.rateLimit.retryAfterSecondsFor(error, retryCount);
|
|
9095
|
+
if (!(args.rateLimit.isRateLimitError?.(error) ?? retryAfterSeconds !== void 0)) return false;
|
|
9096
|
+
const delaySeconds = retryAfterSeconds ?? args.rateLimit.fallbackDelaySecondsForRetry?.(retryCount);
|
|
9097
|
+
if (delaySeconds === void 0 || retryCount > rateLimitMaxRetries) return false;
|
|
9098
|
+
const remainingMs = args.rateLimitState.deadlineMs - rateLimitNow();
|
|
9099
|
+
if (delaySeconds * 1e3 > remainingMs) return false;
|
|
9100
|
+
args.rateLimitState.retryCount = retryCount;
|
|
9101
|
+
await args.rateLimit.onRetry?.({
|
|
9102
|
+
delaySeconds,
|
|
9103
|
+
retryCount
|
|
9104
|
+
});
|
|
9105
|
+
await rateLimitSleep(delaySeconds * 1e3);
|
|
9106
|
+
return true;
|
|
9107
|
+
};
|
|
9108
|
+
const persistPartialCursor = (error) => {
|
|
9109
|
+
const isRateLimit = args.rateLimit?.isRateLimitError?.(error) ?? args.rateLimit?.retryAfterSecondsFor(error, 1) !== void 0;
|
|
9110
|
+
if (!(args.allowPartialResume && args.rateLimit && isRateLimit) || args.collection.pagination.style !== "cursor" || !state.cursor) return false;
|
|
9111
|
+
args.writePendingCursor({
|
|
9112
|
+
collection: args.collection.key,
|
|
9113
|
+
cursor: encodePartialSyncCursor({
|
|
9114
|
+
version: 1,
|
|
9115
|
+
requestIndex: args.requestIndex,
|
|
9116
|
+
cursor: state.cursor,
|
|
9117
|
+
checkpoint,
|
|
9118
|
+
newestId: state.newestId,
|
|
9119
|
+
since: args.since ?? null
|
|
9120
|
+
}),
|
|
9121
|
+
objectCount: upserted,
|
|
9122
|
+
lastSynced: null
|
|
9123
|
+
});
|
|
9124
|
+
return true;
|
|
9125
|
+
};
|
|
8840
9126
|
while (true) {
|
|
8841
9127
|
if (args.pageDelayState.pagesCompleted > 0) {
|
|
8842
9128
|
const delay = pageDelayFor(args.deps.pageDelayMs, args.pageDelayState.pagesCompleted);
|
|
@@ -8863,13 +9149,17 @@ const walkRequest = async (args) => {
|
|
|
8863
9149
|
objectsUpserted: upserted,
|
|
8864
9150
|
creditsSpent: args.credits.spent
|
|
8865
9151
|
};
|
|
8866
|
-
|
|
8867
|
-
|
|
8868
|
-
|
|
8869
|
-
|
|
8870
|
-
|
|
8871
|
-
|
|
8872
|
-
|
|
9152
|
+
let response;
|
|
9153
|
+
while (true) try {
|
|
9154
|
+
response = await args.deps.api.get(path, pageRequestOptions);
|
|
9155
|
+
break;
|
|
9156
|
+
} catch (error) {
|
|
9157
|
+
if (await waitForRateLimit(error)) continue;
|
|
9158
|
+
return wrapPageError({
|
|
9159
|
+
...pageContext,
|
|
9160
|
+
cursorPersisted: persistPartialCursor(error) || void 0
|
|
9161
|
+
}, error);
|
|
9162
|
+
}
|
|
8873
9163
|
const credits = creditsFromHeader(response.headers);
|
|
8874
9164
|
args.credits.spent += credits;
|
|
8875
9165
|
let json;
|
|
@@ -8937,52 +9227,72 @@ const runSync = async (deps, opts) => {
|
|
|
8937
9227
|
let stoppedReason = "complete";
|
|
8938
9228
|
const pendingCursors = [];
|
|
8939
9229
|
const pageDelayState = { pagesCompleted: 0 };
|
|
9230
|
+
const rateLimitNow = opts.rateLimit?.now ?? Date.now;
|
|
9231
|
+
const rateLimitState = opts.rateLimit ? {
|
|
9232
|
+
deadlineMs: opts.rateLimit.timeoutSeconds === void 0 ? void 0 : rateLimitNow() + opts.rateLimit.timeoutSeconds * 1e3,
|
|
9233
|
+
retryCount: 0
|
|
9234
|
+
} : void 0;
|
|
8940
9235
|
const addResult = (result) => {
|
|
8941
9236
|
pages += result.pages;
|
|
8942
9237
|
upserted += result.upserted;
|
|
8943
9238
|
stoppedReason = mergeStoppedReason(stoppedReason, result.stoppedReason);
|
|
8944
9239
|
};
|
|
9240
|
+
const writePendingCursor = (pending) => {
|
|
9241
|
+
deps.cache.setCursor(pending.collection, pending.cursor, pending.objectCount, pending.lastSynced);
|
|
9242
|
+
};
|
|
8945
9243
|
if (collection.parentCollection) {
|
|
8946
9244
|
const parentCollection = collectionByKey(collection.parentCollection);
|
|
8947
9245
|
if (!parentCollection) throw new Error(`Missing parent collection ${collection.parentCollection}`);
|
|
8948
9246
|
const parentIds = [];
|
|
8949
9247
|
const parentPersistsCheckpoint = parentCollection.requests.length === 1;
|
|
8950
|
-
|
|
9248
|
+
let parentCheckpoint;
|
|
9249
|
+
for (const [requestIndex, request] of parentCollection.requests.entries()) {
|
|
8951
9250
|
const result = await walkRequest({
|
|
8952
9251
|
deps,
|
|
8953
9252
|
collection: parentCollection,
|
|
8954
9253
|
request,
|
|
9254
|
+
requestIndex,
|
|
8955
9255
|
since: opts.since,
|
|
8956
9256
|
useCheckpoint: false,
|
|
8957
9257
|
collectIds: true,
|
|
9258
|
+
allowPartialResume: false,
|
|
8958
9259
|
credits,
|
|
8959
9260
|
pageDelayState,
|
|
9261
|
+
rateLimit: opts.rateLimit,
|
|
9262
|
+
rateLimitState,
|
|
9263
|
+
writePendingCursor,
|
|
8960
9264
|
onPage: opts.onPage,
|
|
8961
9265
|
onProgress: opts.onProgress
|
|
8962
9266
|
});
|
|
8963
9267
|
addResult(result);
|
|
8964
9268
|
parentIds.push(...result.ids);
|
|
8965
|
-
if (parentPersistsCheckpoint)
|
|
9269
|
+
if (parentPersistsCheckpoint) parentCheckpoint = singleRequestCheckpoint({
|
|
8966
9270
|
collection: parentCollection,
|
|
8967
9271
|
result
|
|
8968
|
-
})
|
|
9272
|
+
});
|
|
8969
9273
|
}
|
|
8970
|
-
if (!parentPersistsCheckpoint)
|
|
9274
|
+
if (!parentPersistsCheckpoint) parentCheckpoint = {
|
|
8971
9275
|
collection: parentCollection.key,
|
|
8972
9276
|
cursor: null,
|
|
8973
9277
|
objectCount: upserted,
|
|
8974
9278
|
lastSynced: Date.now()
|
|
8975
|
-
}
|
|
8976
|
-
|
|
9279
|
+
};
|
|
9280
|
+
if (parentCheckpoint) writePendingCursor(parentCheckpoint);
|
|
9281
|
+
for (const parentId of parentIds) for (const [requestIndex, request] of collection.requests.entries()) addResult(await walkRequest({
|
|
8977
9282
|
deps,
|
|
8978
9283
|
collection,
|
|
8979
9284
|
request,
|
|
9285
|
+
requestIndex,
|
|
8980
9286
|
since: opts.since,
|
|
8981
9287
|
useCheckpoint: false,
|
|
9288
|
+
allowPartialResume: false,
|
|
8982
9289
|
parentId,
|
|
8983
9290
|
parentIdToken: collection.parentIdToken,
|
|
8984
9291
|
credits,
|
|
8985
9292
|
pageDelayState,
|
|
9293
|
+
rateLimit: opts.rateLimit,
|
|
9294
|
+
rateLimitState,
|
|
9295
|
+
writePendingCursor,
|
|
8986
9296
|
onPage: opts.onPage,
|
|
8987
9297
|
onProgress: opts.onProgress
|
|
8988
9298
|
}));
|
|
@@ -8994,15 +9304,24 @@ const runSync = async (deps, opts) => {
|
|
|
8994
9304
|
});
|
|
8995
9305
|
} else {
|
|
8996
9306
|
const useCheckpoint = collection.requests.length === 1;
|
|
8997
|
-
|
|
9307
|
+
const partialCursor = decodePartialSyncCursor(deps.cache.getCursor(collection.key)?.cursor ?? null);
|
|
9308
|
+
const activePartialCursor = partialCursor && partialCursor.since === (opts.since ?? null) && partialCursor.requestIndex < collection.requests.length ? partialCursor : void 0;
|
|
9309
|
+
for (const [requestIndex, request] of collection.requests.entries()) {
|
|
9310
|
+
if (activePartialCursor && requestIndex < activePartialCursor.requestIndex) continue;
|
|
8998
9311
|
const result = await walkRequest({
|
|
8999
9312
|
deps,
|
|
9000
9313
|
collection,
|
|
9001
9314
|
request,
|
|
9315
|
+
requestIndex,
|
|
9002
9316
|
since: opts.since,
|
|
9003
9317
|
useCheckpoint,
|
|
9318
|
+
resumeCursor: activePartialCursor?.requestIndex === requestIndex ? activePartialCursor : void 0,
|
|
9319
|
+
allowPartialResume: true,
|
|
9004
9320
|
credits,
|
|
9005
9321
|
pageDelayState,
|
|
9322
|
+
rateLimit: opts.rateLimit,
|
|
9323
|
+
rateLimitState,
|
|
9324
|
+
writePendingCursor,
|
|
9006
9325
|
onPage: opts.onPage,
|
|
9007
9326
|
onProgress: opts.onProgress
|
|
9008
9327
|
});
|
|
@@ -9019,7 +9338,7 @@ const runSync = async (deps, opts) => {
|
|
|
9019
9338
|
lastSynced: Date.now()
|
|
9020
9339
|
});
|
|
9021
9340
|
}
|
|
9022
|
-
for (const pending of pendingCursors)
|
|
9341
|
+
for (const pending of pendingCursors) writePendingCursor(pending);
|
|
9023
9342
|
return {
|
|
9024
9343
|
collection: collection.key,
|
|
9025
9344
|
objectsUpserted: upserted,
|
|
@@ -9813,7 +10132,7 @@ pgTable("profiles", {
|
|
|
9813
10132
|
id: text("id").primaryKey().$defaultFn(() => crypto.randomUUID()),
|
|
9814
10133
|
platform: text("platform", { enum: ["linkedin", "x"] }).notNull(),
|
|
9815
10134
|
providerId: text("provider_id").notNull(),
|
|
9816
|
-
|
|
10135
|
+
username: text("username"),
|
|
9817
10136
|
displayName: text("display_name").notNull(),
|
|
9818
10137
|
avatarURL: text("avatar_url"),
|
|
9819
10138
|
headline: text("headline"),
|
|
@@ -9822,7 +10141,7 @@ pgTable("profiles", {
|
|
|
9822
10141
|
updatedAt: timestamptz("updated_at").notNull().defaultNow()
|
|
9823
10142
|
}, (table) => [
|
|
9824
10143
|
uniqueIndex("profiles_platform_provider_id_pkey").on(table.platform, table.providerId),
|
|
9825
|
-
index("
|
|
10144
|
+
index("profiles_platform_username_idx").on(table.platform, table.username),
|
|
9826
10145
|
check("profiles_platform_check", sql`${table.platform} in ('linkedin', 'x')`)
|
|
9827
10146
|
]);
|
|
9828
10147
|
pgTable("feedback_submissions", {
|
|
@@ -9845,7 +10164,7 @@ pgTable("wrapped_runs", {
|
|
|
9845
10164
|
userId: text("user_id").notNull().references(() => user.id, { onDelete: "cascade" }),
|
|
9846
10165
|
connectedAccountId: text("connected_account_id").references(() => connectedAccounts.id, { onDelete: "set null" }),
|
|
9847
10166
|
platform: text("platform", { enum: ["x"] }).notNull().default("x"),
|
|
9848
|
-
|
|
10167
|
+
username: text("username").notNull(),
|
|
9849
10168
|
platformProfileId: text("platform_profile_id"),
|
|
9850
10169
|
status: text("status", { enum: [
|
|
9851
10170
|
"pending",
|
|
@@ -9865,7 +10184,7 @@ pgTable("wrapped_runs", {
|
|
|
9865
10184
|
}, (table) => [
|
|
9866
10185
|
uniqueIndex("wrapped_runs_share_slug_key").on(table.shareSlug),
|
|
9867
10186
|
index("wrapped_runs_user_created_idx").on(table.userId, table.createdAt.desc()),
|
|
9868
|
-
index("
|
|
10187
|
+
index("wrapped_runs_username_done_idx").on(table.username, table.createdAt.desc()).where(sql`${table.status} = 'done'`),
|
|
9869
10188
|
check("wrapped_runs_platform_check", sql`${table.platform} in ('x')`),
|
|
9870
10189
|
check("wrapped_runs_status_check", sql`${table.status} in ('pending', 'running', 'done', 'failed')`)
|
|
9871
10190
|
]);
|
|
@@ -9930,16 +10249,16 @@ const createURL = (baseURL, path) => {
|
|
|
9930
10249
|
const createAbsoluteURL = (baseURL, path) => createURL(baseURL, path).toString();
|
|
9931
10250
|
//#endregion
|
|
9932
10251
|
//#region ../../packages/linkedin/src/profiles.ts
|
|
9933
|
-
const displayNameFrom = (firstName, lastName,
|
|
10252
|
+
const displayNameFrom = (firstName, lastName, username) => [firstName, lastName].filter(Boolean).join(" ").trim() || username || "";
|
|
9934
10253
|
const rowFrom$1 = (fields) => {
|
|
9935
10254
|
const providerId = fields.providerId?.trim();
|
|
9936
10255
|
if (!providerId) return null;
|
|
9937
|
-
const
|
|
9938
|
-
const displayName = fields.displayName?.trim() || displayNameFrom(fields.firstName, fields.lastName,
|
|
10256
|
+
const username = fields.username?.trim() || null;
|
|
10257
|
+
const displayName = fields.displayName?.trim() || displayNameFrom(fields.firstName, fields.lastName, username ?? void 0);
|
|
9939
10258
|
if (!displayName) return null;
|
|
9940
10259
|
return {
|
|
9941
10260
|
providerId,
|
|
9942
|
-
|
|
10261
|
+
username,
|
|
9943
10262
|
displayName,
|
|
9944
10263
|
avatarURL: fields.avatarURL?.trim() || null,
|
|
9945
10264
|
headline: fields.headline?.trim() || null,
|
|
@@ -9971,7 +10290,7 @@ const rowFromUser = ({ user, providerId, raw }) => rowFrom$1({
|
|
|
9971
10290
|
displayName: user.display_name,
|
|
9972
10291
|
firstName: user.first_name,
|
|
9973
10292
|
lastName: user.last_name,
|
|
9974
|
-
|
|
10293
|
+
username: user.public_identifier,
|
|
9975
10294
|
avatarURL: user.public_picture_url,
|
|
9976
10295
|
headline: headlineFromUser(user),
|
|
9977
10296
|
raw
|
|
@@ -9985,7 +10304,7 @@ const extractProfile$1 = (body) => {
|
|
|
9985
10304
|
displayName: parsed.data.display_name,
|
|
9986
10305
|
firstName: parsed.data.first_name,
|
|
9987
10306
|
lastName: parsed.data.last_name,
|
|
9988
|
-
|
|
10307
|
+
username: parsed.data.public_identifier,
|
|
9989
10308
|
avatarURL: parsed.data.public_picture_url,
|
|
9990
10309
|
headline: headlineFromUser(parsed.data),
|
|
9991
10310
|
raw: parsed.data
|
|
@@ -10676,10 +10995,10 @@ const LINKEDIN_OPERATIONS = [
|
|
|
10676
10995
|
capability: "read",
|
|
10677
10996
|
mutates: false,
|
|
10678
10997
|
pagination: {
|
|
10679
|
-
style: "
|
|
10998
|
+
style: "cursor",
|
|
10680
10999
|
limitParam: "limit",
|
|
10681
|
-
|
|
10682
|
-
|
|
11000
|
+
cursorParam: "cursor",
|
|
11001
|
+
cursorPath: "next_cursor",
|
|
10683
11002
|
itemsPath: "data"
|
|
10684
11003
|
},
|
|
10685
11004
|
costBPS: 50,
|
|
@@ -10688,18 +11007,18 @@ const LINKEDIN_OPERATIONS = [
|
|
|
10688
11007
|
},
|
|
10689
11008
|
parameters: [
|
|
10690
11009
|
{
|
|
10691
|
-
name: "
|
|
11010
|
+
name: "limit",
|
|
10692
11011
|
in: "query",
|
|
10693
11012
|
required: false,
|
|
10694
11013
|
kind: "integer",
|
|
10695
|
-
description: "
|
|
11014
|
+
description: "Rows to return (1-100)"
|
|
10696
11015
|
},
|
|
10697
11016
|
{
|
|
10698
|
-
name: "
|
|
11017
|
+
name: "cursor",
|
|
10699
11018
|
in: "query",
|
|
10700
11019
|
required: false,
|
|
10701
|
-
kind: "
|
|
10702
|
-
description: "
|
|
11020
|
+
kind: "string",
|
|
11021
|
+
description: "Pagination cursor"
|
|
10703
11022
|
},
|
|
10704
11023
|
{
|
|
10705
11024
|
name: "identifier",
|
|
@@ -13170,7 +13489,7 @@ const buildRuntimeArgs = (adapter, op) => {
|
|
|
13170
13489
|
};
|
|
13171
13490
|
args.account = {
|
|
13172
13491
|
type: "string",
|
|
13173
|
-
description: "Account selector: @
|
|
13492
|
+
description: "Account selector: @username or profile_id:<id>; overrides config default"
|
|
13174
13493
|
};
|
|
13175
13494
|
if (adapter.isCacheable(op.endpoint)) args.header = {
|
|
13176
13495
|
type: "string",
|
|
@@ -13669,8 +13988,8 @@ const UserProfileInput = object$1({
|
|
|
13669
13988
|
});
|
|
13670
13989
|
const UserProfileOutput = record$1(string$1(), unknown$1());
|
|
13671
13990
|
const UserPostsInput = object$1({
|
|
13672
|
-
offset: union$1([string$1(), number$1()]).optional(),
|
|
13673
13991
|
limit: union$1([string$1(), number$1()]).optional(),
|
|
13992
|
+
cursor: string$1().optional(),
|
|
13674
13993
|
identifier: string$1()
|
|
13675
13994
|
});
|
|
13676
13995
|
const UserPostsOutput = object$1({
|
|
@@ -14093,7 +14412,7 @@ const linkedinRuntimeAdapter = {
|
|
|
14093
14412
|
accountFromOwnAccount: (ownAccount) => ownAccount.account,
|
|
14094
14413
|
ownAccountTargetId: (ownAccount) => ownAccount.providerId,
|
|
14095
14414
|
ownAccountHeader: (ownAccount, selectedAccount) => ownAccount?.accountIdHeader ?? selectedAccount.selector,
|
|
14096
|
-
accountRecordInput: (ownAccount) => ownAccount.account.
|
|
14415
|
+
accountRecordInput: (ownAccount) => ownAccount.account.username ?? ownAccount.providerId,
|
|
14097
14416
|
argAliases: argNameAliasesFrom,
|
|
14098
14417
|
bodyParamArg,
|
|
14099
14418
|
bodyValidationError: ({ bodyFromJSON, bodyParamEntries, bodyParamValues, bodyText, bodyTextArg, displayTextArg, op }) => {
|
|
@@ -14197,6 +14516,8 @@ const DM_INTERACTION_ENDPOINT_BILLING = {
|
|
|
14197
14516
|
billingCategory: DM_INTERACTION_BILLING_CATEGORY,
|
|
14198
14517
|
costBPS: costBPSForCategory$1(DM_INTERACTION_BILLING_CATEGORY)
|
|
14199
14518
|
};
|
|
14519
|
+
const LINKEDIN_RATE_LIMIT_RETRY_LIMIT$1 = 25;
|
|
14520
|
+
const LINKEDIN_RATE_LIMIT_MAX_FALLBACK_SECONDS$1 = 60;
|
|
14200
14521
|
const linkedinSyncPageDelayMs = (pagesCompleted) => {
|
|
14201
14522
|
if (pagesCompleted < 10) return 1e3;
|
|
14202
14523
|
if (pagesCompleted < 20) return 2500;
|
|
@@ -14289,6 +14610,60 @@ const createSyncProgressReporter = (deps, collectionKey) => {
|
|
|
14289
14610
|
stop: () => void 0
|
|
14290
14611
|
};
|
|
14291
14612
|
};
|
|
14613
|
+
const writeRateLimitRetryNotice$1 = (args) => {
|
|
14614
|
+
process.stderr.write(`LinkedIn rate limited; waiting ${args.delaySeconds}s before retry ${args.retryCount}/${LINKEDIN_RATE_LIMIT_RETRY_LIMIT$1}.\n`);
|
|
14615
|
+
};
|
|
14616
|
+
const linkedInRateLimitFallbackDelaySeconds$1 = (retryCount) => Math.min(2 ** Math.max(0, retryCount - 1), LINKEDIN_RATE_LIMIT_MAX_FALLBACK_SECONDS$1);
|
|
14617
|
+
const shellQuote$1 = (value) => /^[A-Za-z0-9_@%+=:,./-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`;
|
|
14618
|
+
const syncRetryCommand$1 = (platform, args) => {
|
|
14619
|
+
const parts = [
|
|
14620
|
+
"social",
|
|
14621
|
+
platform,
|
|
14622
|
+
"sync"
|
|
14623
|
+
];
|
|
14624
|
+
if (args.collection) parts.push(String(args.collection));
|
|
14625
|
+
if (typeof args.account === "string") parts.push("--account", args.account);
|
|
14626
|
+
if (typeof args.since === "string") parts.push("--since", args.since);
|
|
14627
|
+
if (args.timeout != null) parts.push("--timeout", String(args.timeout));
|
|
14628
|
+
return parts.map(shellQuote$1).join(" ");
|
|
14629
|
+
};
|
|
14630
|
+
const rateLimitCauseFor$1 = (error) => {
|
|
14631
|
+
if (isRateLimitError(error)) return error;
|
|
14632
|
+
if (error instanceof SyncPageError && isRateLimitError(error.cause)) return error.cause;
|
|
14633
|
+
};
|
|
14634
|
+
const enrichRateLimitSyncError$1 = (args) => {
|
|
14635
|
+
const cause = rateLimitCauseFor$1(args.error);
|
|
14636
|
+
if (!cause || typeof cause !== "object") return;
|
|
14637
|
+
const retryAfterSeconds = retryAfterSecondsFromRateLimitError(cause);
|
|
14638
|
+
const resumeAt = retryAfterSeconds === void 0 ? void 0 : new Date(Date.now() + retryAfterSeconds * 1e3).toISOString();
|
|
14639
|
+
const retryCommand = syncRetryCommand$1(args.platform, args.commandArgs);
|
|
14640
|
+
const cursorPersisted = args.error instanceof SyncPageError && args.error.cursorPersisted === true;
|
|
14641
|
+
const savedPages = args.error instanceof SyncPageError && args.error.pagesCompleted > 0;
|
|
14642
|
+
const retryAt = resumeAt ? ` at or after ${resumeAt}` : "";
|
|
14643
|
+
Object.assign(cause, {
|
|
14644
|
+
resumeAt,
|
|
14645
|
+
retryCommand,
|
|
14646
|
+
hint: `Rate limited. Wait${retryAt}, then run \`${retryCommand}\`. ${cursorPersisted ? "Already-synced pages are saved; this sync will resume from the saved cursor." : savedPages ? "Already-synced rows are saved, but this collection cannot safely resume the partial page walk; the re-run may re-fetch unfinished pages." : "No sync page completed before the abort."}`,
|
|
14647
|
+
syncResume: {
|
|
14648
|
+
collection: args.collectionKey,
|
|
14649
|
+
command: retryCommand,
|
|
14650
|
+
cursorPersisted,
|
|
14651
|
+
savedPages,
|
|
14652
|
+
behavior: cursorPersisted ? "resume" : "rerun"
|
|
14653
|
+
}
|
|
14654
|
+
});
|
|
14655
|
+
return cause;
|
|
14656
|
+
};
|
|
14657
|
+
const syncRateLimitOptions$1 = (timeoutSeconds) => ({
|
|
14658
|
+
retryAfterSecondsFor: retryAfterSecondsFromRateLimitError,
|
|
14659
|
+
fallbackDelaySecondsForRetry: linkedInRateLimitFallbackDelaySeconds$1,
|
|
14660
|
+
isRateLimitError,
|
|
14661
|
+
...timeoutSeconds === void 0 ? {} : {
|
|
14662
|
+
timeoutSeconds,
|
|
14663
|
+
maxRetries: LINKEDIN_RATE_LIMIT_RETRY_LIMIT$1,
|
|
14664
|
+
onRetry: writeRateLimitRetryNotice$1
|
|
14665
|
+
}
|
|
14666
|
+
});
|
|
14292
14667
|
const buildSync$1 = (deps, platform) => defineCommand({
|
|
14293
14668
|
meta: commandMeta$2({
|
|
14294
14669
|
name: "sync",
|
|
@@ -14344,7 +14719,7 @@ const buildSync$1 = (deps, platform) => defineCommand({
|
|
|
14344
14719
|
},
|
|
14345
14720
|
since: {
|
|
14346
14721
|
type: "string",
|
|
14347
|
-
description: "ISO date
|
|
14722
|
+
description: "ISO date (2026-05-04) or datetime (2026-05-04T00:00:00Z)"
|
|
14348
14723
|
},
|
|
14349
14724
|
reset: {
|
|
14350
14725
|
type: "boolean",
|
|
@@ -14353,6 +14728,10 @@ const buildSync$1 = (deps, platform) => defineCommand({
|
|
|
14353
14728
|
account: {
|
|
14354
14729
|
type: "string",
|
|
14355
14730
|
description: "Account selector; defaults to the first connected account"
|
|
14731
|
+
},
|
|
14732
|
+
timeout: {
|
|
14733
|
+
type: "string",
|
|
14734
|
+
description: "Optional LinkedIn rate-limit wait budget in seconds"
|
|
14356
14735
|
}
|
|
14357
14736
|
},
|
|
14358
14737
|
run: async ({ args }) => {
|
|
@@ -14364,6 +14743,7 @@ const buildSync$1 = (deps, platform) => defineCommand({
|
|
|
14364
14743
|
const accountSelector = requireAccountSelector$1(res.account.selector);
|
|
14365
14744
|
const accountHeaders = { "x-social-account-id": accountSelector };
|
|
14366
14745
|
if (args.credits != null) throw new UsageError("Unsupported option --credits for linkedin sync.");
|
|
14746
|
+
const timeoutSeconds = args.timeout == null ? void 0 : deps.parseIntegerString("--timeout", args.timeout, { min: 1 });
|
|
14367
14747
|
if (!args.collection) {
|
|
14368
14748
|
if (args.reset === true) throw new UsageError("--reset requires a collection");
|
|
14369
14749
|
const cache = openLinkedinCache({
|
|
@@ -14436,12 +14816,20 @@ const buildSync$1 = (deps, platform) => defineCommand({
|
|
|
14436
14816
|
collection,
|
|
14437
14817
|
since,
|
|
14438
14818
|
onPage: progress.onPage,
|
|
14439
|
-
onProgress
|
|
14819
|
+
onProgress,
|
|
14820
|
+
rateLimit: syncRateLimitOptions$1(timeoutSeconds)
|
|
14440
14821
|
});
|
|
14441
14822
|
progress.stop(summary);
|
|
14442
14823
|
deps.printData(summary);
|
|
14443
14824
|
} catch (error) {
|
|
14444
14825
|
progress.error();
|
|
14826
|
+
const rateLimitError = enrichRateLimitSyncError$1({
|
|
14827
|
+
error,
|
|
14828
|
+
platform,
|
|
14829
|
+
commandArgs: args,
|
|
14830
|
+
collectionKey: collection.key
|
|
14831
|
+
});
|
|
14832
|
+
if (rateLimitError) throw rateLimitError;
|
|
14445
14833
|
throw error;
|
|
14446
14834
|
} finally {
|
|
14447
14835
|
cache.close();
|
|
@@ -14475,7 +14863,7 @@ const buildSql$1 = (deps, platform) => defineCommand({
|
|
|
14475
14863
|
formats: ["json"],
|
|
14476
14864
|
projection: false
|
|
14477
14865
|
},
|
|
14478
|
-
examples: [{ command: "social linkedin sql \"SELECT
|
|
14866
|
+
examples: [{ command: "social linkedin sql \"SELECT user_username, user_name, user_headline FROM li_connections WHERE user_headline LIKE '%founder%' LIMIT 20\"" }, { command: "social linkedin sql \"SELECT sender_name, text FROM li_messages ORDER BY created_at DESC LIMIT 20\"" }],
|
|
14479
14867
|
cost: {
|
|
14480
14868
|
metered: false,
|
|
14481
14869
|
unit: "local_cache",
|
|
@@ -14602,7 +14990,7 @@ const buildMark = (deps) => {
|
|
|
14602
14990
|
},
|
|
14603
14991
|
account: {
|
|
14604
14992
|
type: "string",
|
|
14605
|
-
description: "Account selector: @
|
|
14993
|
+
description: "Account selector: @username or profile_id:<id>; overrides config default"
|
|
14606
14994
|
}
|
|
14607
14995
|
},
|
|
14608
14996
|
run: async ({ args, cmd }) => {
|
|
@@ -14713,7 +15101,7 @@ const buildMessageDelete = (deps) => {
|
|
|
14713
15101
|
},
|
|
14714
15102
|
account: {
|
|
14715
15103
|
type: "string",
|
|
14716
|
-
description: "Account selector: @
|
|
15104
|
+
description: "Account selector: @username or profile_id:<id>; overrides config default"
|
|
14717
15105
|
}
|
|
14718
15106
|
},
|
|
14719
15107
|
run: async ({ args, cmd }) => {
|
|
@@ -14804,7 +15192,7 @@ const buildMessageEdit = (deps) => {
|
|
|
14804
15192
|
},
|
|
14805
15193
|
account: {
|
|
14806
15194
|
type: "string",
|
|
14807
|
-
description: "Account selector: @
|
|
15195
|
+
description: "Account selector: @username or profile_id:<id>; overrides config default"
|
|
14808
15196
|
}
|
|
14809
15197
|
},
|
|
14810
15198
|
run: async ({ args, cmd }) => {
|
|
@@ -14868,12 +15256,12 @@ const buildMessage$1 = (deps, sendCommand) => {
|
|
|
14868
15256
|
target: {
|
|
14869
15257
|
type: "positional",
|
|
14870
15258
|
required: true,
|
|
14871
|
-
description: "LinkedIn person or conversation target: @
|
|
15259
|
+
description: "LinkedIn person or conversation target: @username, profile URL, profile_id:<id>, chat_id:<id>, or a messaging thread URL",
|
|
14872
15260
|
valueHint: "target"
|
|
14873
15261
|
},
|
|
14874
15262
|
account: {
|
|
14875
15263
|
type: "string",
|
|
14876
|
-
description: "Account selector: @
|
|
15264
|
+
description: "Account selector: @username or profile_id:<id>; overrides config default"
|
|
14877
15265
|
}
|
|
14878
15266
|
},
|
|
14879
15267
|
run: async ({ args }) => {
|
|
@@ -14957,7 +15345,7 @@ const buildPosts = (liveCommand) => defineCommand({
|
|
|
14957
15345
|
});
|
|
14958
15346
|
}
|
|
14959
15347
|
});
|
|
14960
|
-
const messagesTeachingUsage = "LinkedIn messages live in your local mirror. Read them with: social linkedin sql \"SELECT
|
|
15348
|
+
const messagesTeachingUsage = "LinkedIn messages live in your local mirror. Read them with: social linkedin sql \"SELECT sender_name, text FROM li_messages ORDER BY created_at DESC LIMIT 20\" (after `social linkedin sync messages`). To change read state: social linkedin messages chat_id:<id> mark read|unread";
|
|
14961
15349
|
const buildMessages = (deps) => {
|
|
14962
15350
|
const markCommand = buildMark(deps);
|
|
14963
15351
|
const schemaContract = markCommand.meta.schemaContract;
|
|
@@ -14980,7 +15368,7 @@ const buildMessages = (deps) => {
|
|
|
14980
15368
|
},
|
|
14981
15369
|
account: {
|
|
14982
15370
|
type: "string",
|
|
14983
|
-
description: "Account selector: @
|
|
15371
|
+
description: "Account selector: @username or profile_id:<id>; overrides config default"
|
|
14984
15372
|
}
|
|
14985
15373
|
},
|
|
14986
15374
|
run: async ({ args }) => {
|
|
@@ -15317,12 +15705,12 @@ const XUser = looseObject({
|
|
|
15317
15705
|
const rowFrom = (user) => {
|
|
15318
15706
|
const providerId = user.id.trim();
|
|
15319
15707
|
if (!providerId) return null;
|
|
15320
|
-
const
|
|
15321
|
-
const displayName = user.name?.trim() ||
|
|
15708
|
+
const username = user.username?.trim() || null;
|
|
15709
|
+
const displayName = user.name?.trim() || username || "";
|
|
15322
15710
|
if (!displayName) return null;
|
|
15323
15711
|
return {
|
|
15324
15712
|
providerId,
|
|
15325
|
-
|
|
15713
|
+
username,
|
|
15326
15714
|
displayName,
|
|
15327
15715
|
avatarURL: user.profile_image_url?.trim() || null,
|
|
15328
15716
|
headline: user.description?.trim() || null,
|
|
@@ -15550,7 +15938,7 @@ const X_OPERATIONS = [
|
|
|
15550
15938
|
required: true,
|
|
15551
15939
|
kind: "string",
|
|
15552
15940
|
valueHint: "target",
|
|
15553
|
-
description: "Target profile: @
|
|
15941
|
+
description: "Target profile: @username, profile URL, or profile_id:<id>"
|
|
15554
15942
|
}]
|
|
15555
15943
|
},
|
|
15556
15944
|
{
|
|
@@ -16102,7 +16490,7 @@ const X_OPERATIONS = [
|
|
|
16102
16490
|
required: true,
|
|
16103
16491
|
kind: "string",
|
|
16104
16492
|
valueHint: "target",
|
|
16105
|
-
description: "Target profile: @
|
|
16493
|
+
description: "Target profile: @username, profile URL, or profile_id:<id>. Defaults to the selected account when omitted."
|
|
16106
16494
|
},
|
|
16107
16495
|
{
|
|
16108
16496
|
name: "user.fields",
|
|
@@ -16278,7 +16666,7 @@ const X_OPERATIONS = [
|
|
|
16278
16666
|
required: true,
|
|
16279
16667
|
kind: "string",
|
|
16280
16668
|
valueHint: "target",
|
|
16281
|
-
description: "Target profile: @
|
|
16669
|
+
description: "Target profile: @username, profile URL, or profile_id:<id>. Defaults to the selected account when omitted."
|
|
16282
16670
|
},
|
|
16283
16671
|
{
|
|
16284
16672
|
name: "max_results",
|
|
@@ -16362,7 +16750,7 @@ const X_OPERATIONS = [
|
|
|
16362
16750
|
required: true,
|
|
16363
16751
|
kind: "string",
|
|
16364
16752
|
valueHint: "target",
|
|
16365
|
-
description: "Target profile: @
|
|
16753
|
+
description: "Target profile: @username, profile URL, or profile_id:<id>. Defaults to the selected account when omitted."
|
|
16366
16754
|
},
|
|
16367
16755
|
{
|
|
16368
16756
|
name: "max_results",
|
|
@@ -16453,7 +16841,7 @@ const X_OPERATIONS = [
|
|
|
16453
16841
|
required: true,
|
|
16454
16842
|
kind: "string",
|
|
16455
16843
|
valueHint: "target",
|
|
16456
|
-
description: "Target profile: @
|
|
16844
|
+
description: "Target profile: @username, profile URL, or profile_id:<id>. Defaults to the selected account when omitted."
|
|
16457
16845
|
},
|
|
16458
16846
|
{
|
|
16459
16847
|
name: "max_results",
|
|
@@ -16565,7 +16953,7 @@ const X_OPERATIONS = [
|
|
|
16565
16953
|
required: true,
|
|
16566
16954
|
kind: "string",
|
|
16567
16955
|
valueHint: "target",
|
|
16568
|
-
description: "Target profile: @
|
|
16956
|
+
description: "Target profile: @username, profile URL, or profile_id:<id>. Defaults to the selected account when omitted."
|
|
16569
16957
|
},
|
|
16570
16958
|
{
|
|
16571
16959
|
name: "since_id",
|
|
@@ -16848,7 +17236,7 @@ const X_OPERATIONS = [
|
|
|
16848
17236
|
required: true,
|
|
16849
17237
|
kind: "string",
|
|
16850
17238
|
valueHint: "target",
|
|
16851
|
-
description: "Target profile: @
|
|
17239
|
+
description: "Target profile: @username, profile URL, or profile_id:<id>. Defaults to the selected account when omitted."
|
|
16852
17240
|
},
|
|
16853
17241
|
{
|
|
16854
17242
|
name: "since_id",
|
|
@@ -17852,13 +18240,13 @@ const targetDescriptionFor = (targetKind) => {
|
|
|
17852
18240
|
const kinds = kindsOf(targetKind);
|
|
17853
18241
|
if (kinds.length === 1) {
|
|
17854
18242
|
const [kind] = kinds;
|
|
17855
|
-
if (kind === "profile") return "Target profile: @
|
|
18243
|
+
if (kind === "profile") return "Target profile: @username, profile URL, or profile_id:<id>";
|
|
17856
18244
|
if (kind === "post") return "Target post: a post URL or post_id:<id>";
|
|
17857
18245
|
if (kind === "list") return "Target list: a list URL or list_id:<id>";
|
|
17858
18246
|
if (kind === "chat") return "Target conversation: a chat URL or chat_id:<id>";
|
|
17859
18247
|
return "Target company: a company URL or company_id:<id>";
|
|
17860
18248
|
}
|
|
17861
|
-
return "Target: @
|
|
18249
|
+
return "Target: @username, URL, or a typed ID such as profile_id:<id>.";
|
|
17862
18250
|
};
|
|
17863
18251
|
const commandParameters = (op) => op.parameters.filter((p) => !(p.in === "path" && ownMode(op, p.name) === "implicit"));
|
|
17864
18252
|
const parameterHelpRank = (op, param, index) => {
|
|
@@ -18105,7 +18493,7 @@ const xRuntimeAdapter = {
|
|
|
18105
18493
|
accountFromOwnAccount: (ownAccount) => ownAccount.account,
|
|
18106
18494
|
ownAccountTargetId: (ownAccount) => ownAccount.xUserId,
|
|
18107
18495
|
ownAccountHeader: (_ownAccount, selectedAccount) => selectedAccount.selector,
|
|
18108
|
-
accountRecordInput: (ownAccount) => ownAccount.account.
|
|
18496
|
+
accountRecordInput: (ownAccount) => ownAccount.account.username ?? ownAccount.xUserId,
|
|
18109
18497
|
argAliases,
|
|
18110
18498
|
bodyTextResolvesTarget: resolvesTargetArg,
|
|
18111
18499
|
bodyTextTargetRequired: resolvesTargetArg,
|
|
@@ -18181,6 +18569,50 @@ const syncListRowFor = (collection, shortName, state) => {
|
|
|
18181
18569
|
objectCount: lastSynced === null ? null : state?.objectCount ?? 0
|
|
18182
18570
|
};
|
|
18183
18571
|
};
|
|
18572
|
+
const shellQuote = (value) => /^[A-Za-z0-9_@%+=:,./-]+$/.test(value) ? value : `'${value.replaceAll("'", "'\\''")}'`;
|
|
18573
|
+
const syncRetryCommand = (platform, args) => {
|
|
18574
|
+
const parts = [
|
|
18575
|
+
"social",
|
|
18576
|
+
platform,
|
|
18577
|
+
"sync"
|
|
18578
|
+
];
|
|
18579
|
+
if (args.collection) parts.push(String(args.collection));
|
|
18580
|
+
if (typeof args.account === "string") parts.push("--account", args.account);
|
|
18581
|
+
if (typeof args.since === "string") parts.push("--since", args.since);
|
|
18582
|
+
if (args.timeout != null) parts.push("--timeout", String(args.timeout));
|
|
18583
|
+
return parts.map(shellQuote).join(" ");
|
|
18584
|
+
};
|
|
18585
|
+
const rateLimitCauseFor = (error) => {
|
|
18586
|
+
if (isRateLimitError(error)) return error;
|
|
18587
|
+
if (error instanceof SyncPageError && isRateLimitError(error.cause)) return error.cause;
|
|
18588
|
+
};
|
|
18589
|
+
const enrichRateLimitSyncError = (args) => {
|
|
18590
|
+
const cause = rateLimitCauseFor(args.error);
|
|
18591
|
+
if (!cause || typeof cause !== "object") return;
|
|
18592
|
+
const retryAfterSeconds = retryAfterSecondsFromRateLimitError(cause);
|
|
18593
|
+
const resumeAt = retryAfterSeconds === void 0 ? void 0 : new Date(Date.now() + retryAfterSeconds * 1e3).toISOString();
|
|
18594
|
+
const retryCommand = syncRetryCommand(args.platform, args.commandArgs);
|
|
18595
|
+
const cursorPersisted = args.error instanceof SyncPageError && args.error.cursorPersisted === true;
|
|
18596
|
+
const savedPages = args.error instanceof SyncPageError && args.error.pagesCompleted > 0;
|
|
18597
|
+
const retryAt = resumeAt ? ` at or after ${resumeAt}` : "";
|
|
18598
|
+
Object.assign(cause, {
|
|
18599
|
+
resumeAt,
|
|
18600
|
+
retryCommand,
|
|
18601
|
+
hint: `Rate limited. Wait${retryAt}, then run \`${retryCommand}\`. ${cursorPersisted ? "Already-synced pages are saved; this sync will resume from the saved cursor." : savedPages ? "Already-synced rows are saved, but this collection cannot safely resume the partial page walk; the re-run may re-fetch unfinished pages." : "No sync page completed before the abort."}`,
|
|
18602
|
+
syncResume: {
|
|
18603
|
+
collection: args.collectionKey,
|
|
18604
|
+
command: retryCommand,
|
|
18605
|
+
cursorPersisted,
|
|
18606
|
+
savedPages,
|
|
18607
|
+
behavior: cursorPersisted ? "resume" : "rerun"
|
|
18608
|
+
}
|
|
18609
|
+
});
|
|
18610
|
+
return cause;
|
|
18611
|
+
};
|
|
18612
|
+
const syncRateLimitOptions = () => ({
|
|
18613
|
+
retryAfterSecondsFor: retryAfterSecondsFromRateLimitError,
|
|
18614
|
+
isRateLimitError
|
|
18615
|
+
});
|
|
18184
18616
|
const buildSync = (deps, platform) => defineCommand({
|
|
18185
18617
|
meta: commandMeta$1({
|
|
18186
18618
|
name: "sync",
|
|
@@ -18231,7 +18663,7 @@ const buildSync = (deps, platform) => defineCommand({
|
|
|
18231
18663
|
},
|
|
18232
18664
|
since: {
|
|
18233
18665
|
type: "string",
|
|
18234
|
-
description: "ISO date
|
|
18666
|
+
description: "ISO date (2026-05-04) or datetime (2026-05-04T00:00:00Z)"
|
|
18235
18667
|
},
|
|
18236
18668
|
reset: {
|
|
18237
18669
|
type: "boolean",
|
|
@@ -18240,6 +18672,10 @@ const buildSync = (deps, platform) => defineCommand({
|
|
|
18240
18672
|
account: {
|
|
18241
18673
|
type: "string",
|
|
18242
18674
|
description: "Account selector; defaults to the first connected account"
|
|
18675
|
+
},
|
|
18676
|
+
timeout: {
|
|
18677
|
+
type: "string",
|
|
18678
|
+
description: "Optional rate-limit wait budget in seconds"
|
|
18243
18679
|
}
|
|
18244
18680
|
},
|
|
18245
18681
|
run: async ({ args }) => {
|
|
@@ -18251,6 +18687,7 @@ const buildSync = (deps, platform) => defineCommand({
|
|
|
18251
18687
|
const accountSelector = requireAccountSelector(res.account.selector);
|
|
18252
18688
|
const accountHeaders = { "x-social-account-id": accountSelector };
|
|
18253
18689
|
if (args.credits != null) throw new UsageError("Unsupported option --credits for x sync.");
|
|
18690
|
+
if (args.timeout != null) deps.parseIntegerString("--timeout", args.timeout, { min: 1 });
|
|
18254
18691
|
if (!args.collection) {
|
|
18255
18692
|
if (args.reset === true) throw new UsageError("--reset requires a collection");
|
|
18256
18693
|
const cache = openXCache({
|
|
@@ -18309,9 +18746,19 @@ const buildSync = (deps, platform) => defineCommand({
|
|
|
18309
18746
|
}, {
|
|
18310
18747
|
collection,
|
|
18311
18748
|
since,
|
|
18312
|
-
onProgress
|
|
18749
|
+
onProgress,
|
|
18750
|
+
rateLimit: syncRateLimitOptions()
|
|
18313
18751
|
});
|
|
18314
18752
|
deps.printData(summary);
|
|
18753
|
+
} catch (error) {
|
|
18754
|
+
const rateLimitError = enrichRateLimitSyncError({
|
|
18755
|
+
error,
|
|
18756
|
+
platform,
|
|
18757
|
+
commandArgs: args,
|
|
18758
|
+
collectionKey: collection.key
|
|
18759
|
+
});
|
|
18760
|
+
if (rateLimitError) throw rateLimitError;
|
|
18761
|
+
throw error;
|
|
18315
18762
|
} finally {
|
|
18316
18763
|
cache.close();
|
|
18317
18764
|
}
|
|
@@ -18346,7 +18793,7 @@ const buildSql = (deps, platform) => {
|
|
|
18346
18793
|
formats: ["json"],
|
|
18347
18794
|
projection: false
|
|
18348
18795
|
},
|
|
18349
|
-
examples: [{ command: "social x sql \"SELECT
|
|
18796
|
+
examples: [{ command: "social x sql \"SELECT username, name, followers_count FROM x_followers ORDER BY followers_count DESC LIMIT 20\"" }, { command: "social x sql \"SELECT username, name FROM x_following WHERE verified = 1 ORDER BY followers_count DESC LIMIT 20\"" }],
|
|
18350
18797
|
cost: {
|
|
18351
18798
|
metered: false,
|
|
18352
18799
|
unit: "local_cache",
|
|
@@ -18452,7 +18899,7 @@ const buildTweets = (deps) => {
|
|
|
18452
18899
|
if (args.target !== void 0) args.target = {
|
|
18453
18900
|
...args.target,
|
|
18454
18901
|
required: true,
|
|
18455
|
-
description: "Target profile: @
|
|
18902
|
+
description: "Target profile: @username, profile URL, or profile_id:<id>"
|
|
18456
18903
|
};
|
|
18457
18904
|
return defineCommand({
|
|
18458
18905
|
meta: commandMeta$1({
|
|
@@ -18486,12 +18933,12 @@ const buildMessage = (deps) => {
|
|
|
18486
18933
|
recipients: {
|
|
18487
18934
|
type: "positional",
|
|
18488
18935
|
required: true,
|
|
18489
|
-
description: "Recipient(s): chat URL, chat_id:<id>, @
|
|
18936
|
+
description: "Recipient(s): chat URL, chat_id:<id>, @username, profile URL, or profile_id:<id>; comma-separate profiles for a group",
|
|
18490
18937
|
valueHint: "recipients"
|
|
18491
18938
|
},
|
|
18492
18939
|
account: {
|
|
18493
18940
|
type: "string",
|
|
18494
|
-
description: "Account selector: @
|
|
18941
|
+
description: "Account selector: @username or profile_id:<id>; overrides config default"
|
|
18495
18942
|
}
|
|
18496
18943
|
},
|
|
18497
18944
|
run: async ({ args }) => {
|
|
@@ -18607,7 +19054,7 @@ const buildReplies = (deps) => {
|
|
|
18607
19054
|
},
|
|
18608
19055
|
account: {
|
|
18609
19056
|
type: "string",
|
|
18610
|
-
description: "Account selector: @
|
|
19057
|
+
description: "Account selector: @username or profile_id:<id>; overrides config default"
|
|
18611
19058
|
}
|
|
18612
19059
|
},
|
|
18613
19060
|
run: async ({ args }) => {
|
|
@@ -18777,14 +19224,14 @@ const prepareAccountConnect$1 = async (deps) => {
|
|
|
18777
19224
|
await pollForSeat$2(deps);
|
|
18778
19225
|
};
|
|
18779
19226
|
const normalizeIdentifier$1 = (value) => value.trim().replace(LEADING_AT_PATTERN$1, "").toLowerCase();
|
|
18780
|
-
const
|
|
19227
|
+
const accountUsername = (account) => account.username.replace(LEADING_AT_PATTERN$1, "");
|
|
18781
19228
|
const linkedinReconnectTargetFrom = (value) => {
|
|
18782
19229
|
const trimmed = value.trim();
|
|
18783
19230
|
if (trimmed.startsWith("@")) {
|
|
18784
|
-
const
|
|
18785
|
-
if (
|
|
18786
|
-
type: "
|
|
18787
|
-
value:
|
|
19231
|
+
const username = normalizeIdentifier$1(trimmed);
|
|
19232
|
+
if (username) return {
|
|
19233
|
+
type: "username",
|
|
19234
|
+
value: username
|
|
18788
19235
|
};
|
|
18789
19236
|
}
|
|
18790
19237
|
if (trimmed.startsWith("profile_id:")) {
|
|
@@ -18799,20 +19246,20 @@ const linkedinReconnectTargetFrom = (value) => {
|
|
|
18799
19246
|
if ((url.protocol === "https:" || url.protocol === "http:") && LINKEDIN_PROFILE_HOST_PATTERN.test(url.hostname.replace(/^www\./i, ""))) {
|
|
18800
19247
|
const [, section, publicIdentifier] = url.pathname.split("/");
|
|
18801
19248
|
if (section === "in" && publicIdentifier) return {
|
|
18802
|
-
type: "
|
|
19249
|
+
type: "username",
|
|
18803
19250
|
value: decodeURIComponent(publicIdentifier).toLowerCase()
|
|
18804
19251
|
};
|
|
18805
19252
|
}
|
|
18806
19253
|
} catch {}
|
|
18807
|
-
throw new Error("LinkedIn reconnect account must be @
|
|
19254
|
+
throw new Error("LinkedIn reconnect account must be @username, a LinkedIn profile URL, or profile_id:<id>.");
|
|
18808
19255
|
};
|
|
18809
19256
|
const accountMatchesReconnectTarget = (account, target) => {
|
|
18810
19257
|
if (target.type === "profileId") return account.profileId.toLowerCase() === target.value;
|
|
18811
|
-
return normalizeIdentifier$1(account.
|
|
19258
|
+
return normalizeIdentifier$1(account.username) === target.value;
|
|
18812
19259
|
};
|
|
18813
19260
|
const reconnectCommandTargetFor = (account) => {
|
|
18814
|
-
const
|
|
18815
|
-
return normalizeIdentifier$1(
|
|
19261
|
+
const username = accountUsername(account);
|
|
19262
|
+
return normalizeIdentifier$1(username) === normalizeIdentifier$1(account.profileId) ? `profile_id:${account.profileId}` : `@${username}`;
|
|
18816
19263
|
};
|
|
18817
19264
|
const listLinkedinLifecycleAccounts = (deps, includeDisconnected = true) => deps.client.listAccounts({
|
|
18818
19265
|
platform: LINKEDIN_PLATFORM,
|
|
@@ -18872,7 +19319,7 @@ const reconnectLinkedinAccount = async (deps, identifier) => {
|
|
|
18872
19319
|
return {
|
|
18873
19320
|
platform: LINKEDIN_PLATFORM,
|
|
18874
19321
|
status: "connected",
|
|
18875
|
-
account: await pollForAccount$1(deps, (candidate) => candidate.profileId === account.profileId && candidate.status === CONNECTED_STATUS, `Timed out waiting for LinkedIn account
|
|
19322
|
+
account: await pollForAccount$1(deps, (candidate) => candidate.profileId === account.profileId && candidate.status === CONNECTED_STATUS, `Timed out waiting for LinkedIn account @${accountUsername(account)} to reconnect.`)
|
|
18876
19323
|
};
|
|
18877
19324
|
};
|
|
18878
19325
|
const disconnectLinkedinAccount = async (deps, identifier) => {
|
|
@@ -18947,7 +19394,7 @@ const prepareAccountConnect = async (deps) => {
|
|
|
18947
19394
|
};
|
|
18948
19395
|
const accountKey = (account) => account.profileId;
|
|
18949
19396
|
const normalizeIdentifier = (value) => value.trim().replace(/^@+/, "").toLowerCase();
|
|
18950
|
-
const accountSelectorFor$1 = (account) => account.
|
|
19397
|
+
const accountSelectorFor$1 = (account) => account.username ? `@${account.username.replace(/^@+/, "")}` : `profile_id:${account.profileId}`;
|
|
18951
19398
|
const assertXAccountSelector = (target) => {
|
|
18952
19399
|
const trimmed = target.trim();
|
|
18953
19400
|
if (trimmed.startsWith("profile_id:")) {
|
|
@@ -18955,7 +19402,7 @@ const assertXAccountSelector = (target) => {
|
|
|
18955
19402
|
if (profileId && !/[/?#%]/.test(profileId)) return;
|
|
18956
19403
|
}
|
|
18957
19404
|
if (trimmed.startsWith("@") && !/[/?#%]/.test(trimmed)) return;
|
|
18958
|
-
throw new Error("account must be an @
|
|
19405
|
+
throw new Error("account must be an @username or profile_id:<id>.");
|
|
18959
19406
|
};
|
|
18960
19407
|
const findAccount = (accounts, target) => {
|
|
18961
19408
|
const trimmed = target.trim();
|
|
@@ -18963,7 +19410,7 @@ const findAccount = (accounts, target) => {
|
|
|
18963
19410
|
const profileId = trimmed.slice(11).trim().toLowerCase();
|
|
18964
19411
|
return accounts.find((account) => account.profileId.toLowerCase() === profileId);
|
|
18965
19412
|
}
|
|
18966
|
-
if (trimmed.startsWith("@")) return accounts.find((account) => normalizeIdentifier(account.
|
|
19413
|
+
if (trimmed.startsWith("@")) return accounts.find((account) => normalizeIdentifier(account.username) === normalizeIdentifier(target));
|
|
18967
19414
|
};
|
|
18968
19415
|
const listXLifecycleAccounts = (deps, includeDisconnected) => deps.client.accounts.list({
|
|
18969
19416
|
platform: "x",
|
|
@@ -21572,7 +22019,7 @@ const createInstance = (defaults) => {
|
|
|
21572
22019
|
const ky = createInstance();
|
|
21573
22020
|
//#endregion
|
|
21574
22021
|
//#region package.json
|
|
21575
|
-
var version$1 = "0.
|
|
22022
|
+
var version$1 = "0.8.0";
|
|
21576
22023
|
//#endregion
|
|
21577
22024
|
//#region src/lib/env.ts
|
|
21578
22025
|
const URLWithTrailingSlash = url().transform(ensureTrailingSlash);
|
|
@@ -21705,12 +22152,20 @@ var CLIAPIError = class extends Error {
|
|
|
21705
22152
|
status;
|
|
21706
22153
|
body;
|
|
21707
22154
|
retryAfterSeconds;
|
|
21708
|
-
|
|
22155
|
+
hint;
|
|
22156
|
+
resumeAt;
|
|
22157
|
+
retryCommand;
|
|
22158
|
+
syncResume;
|
|
22159
|
+
constructor(message, status, body, retryAfterSeconds, details = {}) {
|
|
21709
22160
|
super(message);
|
|
21710
22161
|
this.name = "CLIAPIError";
|
|
21711
22162
|
this.status = status;
|
|
21712
22163
|
this.body = body;
|
|
21713
22164
|
this.retryAfterSeconds = retryAfterSeconds;
|
|
22165
|
+
this.hint = details.hint;
|
|
22166
|
+
this.resumeAt = details.resumeAt;
|
|
22167
|
+
this.retryCommand = details.retryCommand;
|
|
22168
|
+
this.syncResume = details.syncResume;
|
|
21714
22169
|
}
|
|
21715
22170
|
};
|
|
21716
22171
|
const readBearerToken = async () => {
|
|
@@ -21720,8 +22175,12 @@ const readBearerToken = async () => {
|
|
|
21720
22175
|
};
|
|
21721
22176
|
const linkedInRateLimitFallbackDelaySeconds = (retryCount) => Math.min(2 ** Math.max(0, retryCount - 1), LINKEDIN_RATE_LIMIT_MAX_FALLBACK_SECONDS);
|
|
21722
22177
|
const linkedInRateLimitRetryDelaySeconds = (retryCount, retryAfterSeconds) => retryAfterSeconds ?? linkedInRateLimitFallbackDelaySeconds(retryCount);
|
|
22178
|
+
const linkedInRateLimitWaitBudgetCovers = (args) => args.delaySeconds <= (args.waitBudgetSeconds ?? LINKEDIN_RATE_LIMIT_MAX_FALLBACK_SECONDS);
|
|
21723
22179
|
const isLinkedInProxyRequest = (request) => new URL(request.url).pathname.includes("/linkedin/");
|
|
21724
|
-
const shouldRetryLinkedInRateLimit = (args) => args.error instanceof HTTPError && args.error.response.status === 429 && isLinkedInProxyRequest(args.error.request) && args.retryCount <= LINKEDIN_RATE_LIMIT_RETRY_LIMIT
|
|
22180
|
+
const shouldRetryLinkedInRateLimit = (args) => args.error instanceof HTTPError && args.error.response.status === 429 && isLinkedInProxyRequest(args.error.request) && args.retryCount <= LINKEDIN_RATE_LIMIT_RETRY_LIMIT && linkedInRateLimitWaitBudgetCovers({
|
|
22181
|
+
delaySeconds: linkedInRateLimitRetryDelaySeconds(args.retryCount, retryAfterSecondsFrom(args.error.response.headers.get("retry-after"))),
|
|
22182
|
+
waitBudgetSeconds: args.waitBudgetSeconds
|
|
22183
|
+
});
|
|
21725
22184
|
const writeRateLimitRetryNotice = (args) => {
|
|
21726
22185
|
process.stderr.write(`LinkedIn rate limited; waiting ${args.delaySeconds}s before retry ${args.retryCount}/${LINKEDIN_RATE_LIMIT_RETRY_LIMIT}.\n`);
|
|
21727
22186
|
};
|
|
@@ -21913,26 +22372,9 @@ const authWhoamiOutput = async (client, apiURL = env.SOCIAL_API_URL) => {
|
|
|
21913
22372
|
};
|
|
21914
22373
|
}
|
|
21915
22374
|
};
|
|
21916
|
-
|
|
21917
|
-
|
|
21918
|
-
const
|
|
21919
|
-
live: 900,
|
|
21920
|
-
analytical: 1440 * 60,
|
|
21921
|
-
historical: 10080 * 60
|
|
21922
|
-
};
|
|
21923
|
-
const CACHE_MODES = [
|
|
21924
|
-
"live",
|
|
21925
|
-
"analytical",
|
|
21926
|
-
"historical"
|
|
21927
|
-
];
|
|
21928
|
-
const MAX_CACHE_TTL_SECONDS = CACHE_MODE_TTL_SECONDS.historical;
|
|
21929
|
-
const CacheMode = _enum$1([...CACHE_MODES, "custom"]);
|
|
21930
|
-
const ConfigurableCacheMode = _enum$1(CACHE_MODES);
|
|
21931
|
-
const CLIConfig = object$1({ cache: object$1({
|
|
21932
|
-
mode: CacheMode.default("live"),
|
|
21933
|
-
cacheTTLSeconds: number$1().int().min(0).max(MAX_CACHE_TTL_SECONDS).optional()
|
|
21934
|
-
}).default({ mode: "live" }) });
|
|
21935
|
-
const defaultCLIConfig = () => ({ cache: { mode: "live" } });
|
|
22375
|
+
const MAX_CACHE_TTL_SECONDS = 10080 * 60;
|
|
22376
|
+
const CLIConfig = object$1({ cache: object$1({ cacheTTLSeconds: number$1().int().min(0).max(MAX_CACHE_TTL_SECONDS).default(900) }).default({ cacheTTLSeconds: 900 }) });
|
|
22377
|
+
const defaultCLIConfig = () => ({ cache: { cacheTTLSeconds: 900 } });
|
|
21936
22378
|
const configPath = () => join(process.env.HOME ?? homedir(), ".social", "config.json");
|
|
21937
22379
|
const parseCLIConfig = (serialized) => CLIConfig.parse(JSON.parse(serialized));
|
|
21938
22380
|
const readCLIConfig = async () => {
|
|
@@ -21950,27 +22392,15 @@ const writeCLIConfig = async (config) => {
|
|
|
21950
22392
|
await writeFile(path, `${JSON.stringify(parsed, null, 2)}\n`, { mode: 384 });
|
|
21951
22393
|
await chmod(path, 384);
|
|
21952
22394
|
};
|
|
21953
|
-
const cacheTTLSecondsFrom = (config) => config.cache.mode === "custom" ? config.cache.cacheTTLSeconds ?? CACHE_MODE_TTL_SECONDS.live : CACHE_MODE_TTL_SECONDS[config.cache.mode];
|
|
21954
|
-
const setCacheMode = async (mode) => {
|
|
21955
|
-
const next = {
|
|
21956
|
-
...await readCLIConfig(),
|
|
21957
|
-
cache: { mode }
|
|
21958
|
-
};
|
|
21959
|
-
await writeCLIConfig(next);
|
|
21960
|
-
return next;
|
|
21961
|
-
};
|
|
21962
22395
|
const setCacheTTLSeconds = async (cacheTTLSeconds) => {
|
|
21963
22396
|
const next = {
|
|
21964
22397
|
...await readCLIConfig(),
|
|
21965
|
-
cache: {
|
|
21966
|
-
mode: "custom",
|
|
21967
|
-
cacheTTLSeconds
|
|
21968
|
-
}
|
|
22398
|
+
cache: { cacheTTLSeconds }
|
|
21969
22399
|
};
|
|
21970
22400
|
await writeCLIConfig(next);
|
|
21971
22401
|
return next;
|
|
21972
22402
|
};
|
|
21973
|
-
const cacheControlHeaderFromConfig = async () => `max-age=${
|
|
22403
|
+
const cacheControlHeaderFromConfig = async () => `max-age=${(await readCLIConfig()).cache.cacheTTLSeconds}`;
|
|
21974
22404
|
//#endregion
|
|
21975
22405
|
//#region src/lib/output.ts
|
|
21976
22406
|
const isRecord$3 = (value) => Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
@@ -22159,7 +22589,7 @@ const resultEnvelope = ({ account, contract, data, local, resolved = [], respons
|
|
|
22159
22589
|
return {
|
|
22160
22590
|
account: {
|
|
22161
22591
|
platform: account.platform,
|
|
22162
|
-
|
|
22592
|
+
username: account.username
|
|
22163
22593
|
},
|
|
22164
22594
|
...payload,
|
|
22165
22595
|
meta: {
|
|
@@ -22189,7 +22619,11 @@ const errorPayload = (error) => {
|
|
|
22189
22619
|
type: error.name,
|
|
22190
22620
|
status: error.status,
|
|
22191
22621
|
body: error.body,
|
|
22192
|
-
retryAfterSeconds: error.retryAfterSeconds
|
|
22622
|
+
retryAfterSeconds: error.retryAfterSeconds,
|
|
22623
|
+
resumeAt: error.resumeAt,
|
|
22624
|
+
retryCommand: error.retryCommand,
|
|
22625
|
+
hint: error.hint,
|
|
22626
|
+
syncResume: error.syncResume
|
|
22193
22627
|
};
|
|
22194
22628
|
if (error instanceof Error) return {
|
|
22195
22629
|
error: error.message,
|
|
@@ -22213,7 +22647,11 @@ const apiErrorCause = (error) => ({
|
|
|
22213
22647
|
error: error.message,
|
|
22214
22648
|
status: error.status,
|
|
22215
22649
|
body: error.body,
|
|
22216
|
-
retryAfterSeconds: error.retryAfterSeconds
|
|
22650
|
+
retryAfterSeconds: error.retryAfterSeconds,
|
|
22651
|
+
resumeAt: error.resumeAt,
|
|
22652
|
+
retryCommand: error.retryCommand,
|
|
22653
|
+
hint: error.hint,
|
|
22654
|
+
syncResume: error.syncResume
|
|
22217
22655
|
});
|
|
22218
22656
|
const errorCause = (error) => {
|
|
22219
22657
|
if (!("cause" in error)) return {};
|
|
@@ -22287,7 +22725,7 @@ const TARGET_PREFIXES = {
|
|
|
22287
22725
|
request_id: "request"
|
|
22288
22726
|
};
|
|
22289
22727
|
const KIND_HINTS = {
|
|
22290
|
-
profile: "a profile (@
|
|
22728
|
+
profile: "a profile (@username, a profile URL, or profile_id:…)",
|
|
22291
22729
|
post: "a post (a post URL or post_id:…)",
|
|
22292
22730
|
list: "a list (a list URL or list_id:…)",
|
|
22293
22731
|
chat: "a conversation (a chat URL or chat_id:…)",
|
|
@@ -22399,8 +22837,8 @@ const resolveTarget = async (name, value, options) => {
|
|
|
22399
22837
|
const list = url.pathname.match(/^\/i\/lists\/(\d+)/)?.[1];
|
|
22400
22838
|
if (list) return accept("list", list, { url: url.toString() });
|
|
22401
22839
|
const segments = url.pathname.split("/").filter(Boolean);
|
|
22402
|
-
const
|
|
22403
|
-
if (segments.length === 1 &&
|
|
22840
|
+
const username = segments[0];
|
|
22841
|
+
if (segments.length === 1 && username && !X_RESERVED_SEGMENTS.has(username.toLowerCase())) return await lookupProfile(username, url.toString());
|
|
22404
22842
|
}
|
|
22405
22843
|
if (platform === "linkedin" && isLinkedinHost(url)) {
|
|
22406
22844
|
const post = linkedinPostURN(input, url);
|
|
@@ -22421,10 +22859,10 @@ const resolveTarget = async (name, value, options) => {
|
|
|
22421
22859
|
if (colon > 0) {
|
|
22422
22860
|
const prefix = input.slice(0, colon);
|
|
22423
22861
|
const kind = TARGET_PREFIXES[prefix];
|
|
22424
|
-
if (!kind) throw validationError([name], `${name}: unknown prefix "${prefix}:". Use ${Object.keys(TARGET_PREFIXES).map((targetPrefix) => `${targetPrefix}:<id>`).join(", ")}, @
|
|
22862
|
+
if (!kind) throw validationError([name], `${name}: unknown prefix "${prefix}:". Use ${Object.keys(TARGET_PREFIXES).map((targetPrefix) => `${targetPrefix}:<id>`).join(", ")}, @username, or a URL.`);
|
|
22425
22863
|
return accept(kind, parseResourceId(name, input.slice(colon + 1)));
|
|
22426
22864
|
}
|
|
22427
|
-
throw validationError([name], `${name} must be @
|
|
22865
|
+
throw validationError([name], `${name} must be @username, a URL, or a typed ID (${Object.keys(TARGET_PREFIXES).map((targetPrefix) => `${targetPrefix}:<id>`).join(", ")}). Bare IDs are not accepted.`);
|
|
22428
22866
|
};
|
|
22429
22867
|
const plainResourceRecord = (platform, kind, id, options = {}) => resolutionRecord({
|
|
22430
22868
|
input: options.input ?? id,
|
|
@@ -22493,14 +22931,8 @@ const commandMeta = (meta) => {
|
|
|
22493
22931
|
contract
|
|
22494
22932
|
};
|
|
22495
22933
|
};
|
|
22496
|
-
const CacheStateOutput = object$1({
|
|
22497
|
-
|
|
22498
|
-
cacheTTLSeconds: number$1().int().nonnegative()
|
|
22499
|
-
});
|
|
22500
|
-
const cacheStateFrom = (config) => ({
|
|
22501
|
-
mode: config.cache.mode,
|
|
22502
|
-
cacheTTLSeconds: cacheTTLSecondsFrom(config)
|
|
22503
|
-
});
|
|
22934
|
+
const CacheStateOutput = object$1({ cacheTTLSeconds: number$1().int().nonnegative() });
|
|
22935
|
+
const cacheStateFrom = (config) => ({ cacheTTLSeconds: config.cache.cacheTTLSeconds });
|
|
22504
22936
|
const printCacheState = (config) => {
|
|
22505
22937
|
printData(cacheStateFrom(config));
|
|
22506
22938
|
};
|
|
@@ -22514,79 +22946,43 @@ const configCommand = defineCommand({
|
|
|
22514
22946
|
name: "cache",
|
|
22515
22947
|
description: "Read or configure proxy cache defaults."
|
|
22516
22948
|
},
|
|
22517
|
-
subCommands: {
|
|
22518
|
-
|
|
22519
|
-
|
|
22520
|
-
|
|
22521
|
-
|
|
22522
|
-
|
|
22523
|
-
|
|
22524
|
-
|
|
22525
|
-
|
|
22526
|
-
|
|
22527
|
-
|
|
22528
|
-
|
|
22529
|
-
|
|
22530
|
-
|
|
22531
|
-
|
|
22532
|
-
|
|
22533
|
-
|
|
22534
|
-
|
|
22535
|
-
|
|
22536
|
-
}]
|
|
22537
|
-
}
|
|
22538
|
-
}),
|
|
22539
|
-
args: { mode: {
|
|
22540
|
-
type: "positional",
|
|
22541
|
-
required: false,
|
|
22542
|
-
description: "Cache mode: live, analytical, or historical."
|
|
22543
|
-
} },
|
|
22544
|
-
run: async ({ args }) => {
|
|
22545
|
-
if (args.mode === void 0) {
|
|
22546
|
-
printCacheState(await readCLIConfig());
|
|
22547
|
-
return;
|
|
22548
|
-
}
|
|
22549
|
-
printCacheState(await setCacheMode(ConfigurableCacheMode.parse(String(args.mode))));
|
|
22949
|
+
subCommands: { ttl: defineCommand({
|
|
22950
|
+
meta: commandMeta({
|
|
22951
|
+
name: "ttl",
|
|
22952
|
+
description: "Read or set the default cache TTL in seconds.",
|
|
22953
|
+
contract: {
|
|
22954
|
+
capability: "none",
|
|
22955
|
+
auth: {
|
|
22956
|
+
local: true,
|
|
22957
|
+
required: false
|
|
22958
|
+
},
|
|
22959
|
+
mutates: true,
|
|
22960
|
+
outputSchema: CacheStateOutput,
|
|
22961
|
+
idempotency: "idempotent",
|
|
22962
|
+
confirmation: false,
|
|
22963
|
+
responseShaping: { formats: ["json"] },
|
|
22964
|
+
hazards: [{
|
|
22965
|
+
code: "local_config_write",
|
|
22966
|
+
message: "Writes local CLI configuration when ttl_seconds is provided."
|
|
22967
|
+
}]
|
|
22550
22968
|
}
|
|
22551
22969
|
}),
|
|
22552
|
-
|
|
22553
|
-
|
|
22554
|
-
|
|
22555
|
-
|
|
22556
|
-
|
|
22557
|
-
|
|
22558
|
-
|
|
22559
|
-
|
|
22560
|
-
|
|
22561
|
-
},
|
|
22562
|
-
mutates: true,
|
|
22563
|
-
outputSchema: CacheStateOutput,
|
|
22564
|
-
idempotency: "idempotent",
|
|
22565
|
-
confirmation: false,
|
|
22566
|
-
responseShaping: { formats: ["json"] },
|
|
22567
|
-
hazards: [{
|
|
22568
|
-
code: "local_config_write",
|
|
22569
|
-
message: "Writes local CLI configuration when ttl_seconds is provided."
|
|
22570
|
-
}]
|
|
22571
|
-
}
|
|
22572
|
-
}),
|
|
22573
|
-
args: { ttl_seconds: {
|
|
22574
|
-
type: "positional",
|
|
22575
|
-
required: false,
|
|
22576
|
-
description: "Cache TTL in seconds. Setting this switches cache mode to custom."
|
|
22577
|
-
} },
|
|
22578
|
-
run: async ({ args }) => {
|
|
22579
|
-
if (args.ttl_seconds === void 0) {
|
|
22580
|
-
printCacheState(await readCLIConfig());
|
|
22581
|
-
return;
|
|
22582
|
-
}
|
|
22583
|
-
printCacheState(await setCacheTTLSeconds(parseIntegerString("--ttl-seconds", args.ttl_seconds, {
|
|
22584
|
-
min: 0,
|
|
22585
|
-
max: MAX_CACHE_TTL_SECONDS
|
|
22586
|
-
})));
|
|
22970
|
+
args: { ttl_seconds: {
|
|
22971
|
+
type: "positional",
|
|
22972
|
+
required: false,
|
|
22973
|
+
description: "Cache TTL in seconds."
|
|
22974
|
+
} },
|
|
22975
|
+
run: async ({ args }) => {
|
|
22976
|
+
if (args.ttl_seconds === void 0) {
|
|
22977
|
+
printCacheState(await readCLIConfig());
|
|
22978
|
+
return;
|
|
22587
22979
|
}
|
|
22588
|
-
|
|
22589
|
-
|
|
22980
|
+
printCacheState(await setCacheTTLSeconds(parseIntegerString("--ttl-seconds", args.ttl_seconds, {
|
|
22981
|
+
min: 0,
|
|
22982
|
+
max: MAX_CACHE_TTL_SECONDS
|
|
22983
|
+
})));
|
|
22984
|
+
}
|
|
22985
|
+
}) }
|
|
22590
22986
|
}) }
|
|
22591
22987
|
});
|
|
22592
22988
|
//#endregion
|
|
@@ -28871,7 +29267,7 @@ const writeAuthContract = {
|
|
|
28871
29267
|
};
|
|
28872
29268
|
const LifecycleAccountOutput = object$1({
|
|
28873
29269
|
platform: string$1(),
|
|
28874
|
-
|
|
29270
|
+
username: string$1(),
|
|
28875
29271
|
profileId: string$1(),
|
|
28876
29272
|
url: string$1(),
|
|
28877
29273
|
displayName: string$1(),
|
|
@@ -28937,17 +29333,17 @@ const AccountOverviewOutput = object$1({
|
|
|
28937
29333
|
const accountArg = {
|
|
28938
29334
|
type: "positional",
|
|
28939
29335
|
required: true,
|
|
28940
|
-
description: "Account @
|
|
29336
|
+
description: "Account @username, profile URL, or profile_id:<id>."
|
|
28941
29337
|
};
|
|
28942
29338
|
const xAccountArg = {
|
|
28943
29339
|
type: "positional",
|
|
28944
29340
|
required: true,
|
|
28945
|
-
description: "X account @
|
|
29341
|
+
description: "X account @username or profile_id:<id>."
|
|
28946
29342
|
};
|
|
28947
29343
|
const linkedinReconnectAccountArg = {
|
|
28948
29344
|
type: "positional",
|
|
28949
29345
|
required: true,
|
|
28950
|
-
description: "LinkedIn account @
|
|
29346
|
+
description: "LinkedIn account @username, profile URL, or profile_id:<id>."
|
|
28951
29347
|
};
|
|
28952
29348
|
const writeOutput = async (deps, value) => {
|
|
28953
29349
|
if (deps.writeOutput) {
|
|
@@ -28973,7 +29369,7 @@ const lifecycleConnectContract = {
|
|
|
28973
29369
|
idempotency: "non-idempotent",
|
|
28974
29370
|
confirmation: false
|
|
28975
29371
|
};
|
|
28976
|
-
const
|
|
29372
|
+
const usernameLabelFor = (account) => account.username.startsWith("@") ? account.username : `@${account.username}`;
|
|
28977
29373
|
const authHandshakeMessage = "Handshaking auth...";
|
|
28978
29374
|
const lifecycleDepsWithTextLogs = (deps) => ({
|
|
28979
29375
|
...deps,
|
|
@@ -28998,7 +29394,7 @@ const lifecycleDepsWithTextLogs = (deps) => ({
|
|
|
28998
29394
|
}
|
|
28999
29395
|
});
|
|
29000
29396
|
const runConnectOutput = async (deps, connect) => {
|
|
29001
|
-
writeText(deps, `${
|
|
29397
|
+
writeText(deps, `${usernameLabelFor((await connect(lifecycleDepsWithTextLogs(deps))).account)} connected!`);
|
|
29002
29398
|
};
|
|
29003
29399
|
const createConnectPlatformCommand = (platform, run) => defineCommand({
|
|
29004
29400
|
meta: commandMeta({
|
|
@@ -29206,11 +29602,11 @@ const createFeedbackCommand = (deps) => defineCommand({
|
|
|
29206
29602
|
//#region src/lib/account.ts
|
|
29207
29603
|
const LEADING_AT_PATTERN = /^@+/;
|
|
29208
29604
|
const normalizeAccountIdentifier = (value) => value.trim().replace(LEADING_AT_PATTERN, "").toLowerCase();
|
|
29209
|
-
const accountSelectorFor = (account) => account.
|
|
29605
|
+
const accountSelectorFor = (account) => account.username ? `@${account.username.replace(LEADING_AT_PATTERN, "")}` : `profile_id:${account.profileId}`;
|
|
29210
29606
|
const accountMatchesIdentifier = (account, identifier) => {
|
|
29211
29607
|
const trimmed = identifier.trim();
|
|
29212
29608
|
if (trimmed.startsWith("profile_id:")) return account.profileId.toLowerCase() === trimmed.slice(11).trim().toLowerCase();
|
|
29213
|
-
return trimmed.startsWith("@") && normalizeAccountIdentifier(account.
|
|
29609
|
+
return trimmed.startsWith("@") && normalizeAccountIdentifier(account.username) === normalizeAccountIdentifier(trimmed);
|
|
29214
29610
|
};
|
|
29215
29611
|
const resolveAccountId = ({ account }) => {
|
|
29216
29612
|
if (!account) return;
|
|
@@ -29306,12 +29702,12 @@ const createCLIDeps = () => {
|
|
|
29306
29702
|
if (account && !selected) throw new NotFoundError("account_not_found");
|
|
29307
29703
|
return {
|
|
29308
29704
|
platform,
|
|
29309
|
-
|
|
29705
|
+
username: selected?.username ?? (account?.startsWith("@") ? account.slice(1) : account ?? "default"),
|
|
29310
29706
|
selector: selected ? accountSelectorFor(selected) : account
|
|
29311
29707
|
};
|
|
29312
29708
|
};
|
|
29313
29709
|
const lookupXHandle = async (input, account) => {
|
|
29314
|
-
const
|
|
29710
|
+
const inputUsername = input.replace(/^@+/, "");
|
|
29315
29711
|
const selectedAccount = await resolveAccount({
|
|
29316
29712
|
account,
|
|
29317
29713
|
platform: "x"
|
|
@@ -29320,13 +29716,13 @@ const createCLIDeps = () => {
|
|
|
29320
29716
|
"Cache-Control": await cacheControlHeaderFromConfig(),
|
|
29321
29717
|
...selectedAccount.selector === void 0 ? {} : { "x-social-account-id": selectedAccount.selector }
|
|
29322
29718
|
};
|
|
29323
|
-
const data = await api.get(`x/users/by/username/${encodeURIComponent(
|
|
29324
|
-
if (!data || typeof data !== "object" || !("data" in data)) throw new Error(`Could not resolve X
|
|
29719
|
+
const data = await api.get(`x/users/by/username/${encodeURIComponent(inputUsername)}`, { headers }).json();
|
|
29720
|
+
if (!data || typeof data !== "object" || !("data" in data)) throw new Error(`Could not resolve X username ${input}.`);
|
|
29325
29721
|
const user = data.data;
|
|
29326
|
-
if (!user || typeof user !== "object" || !("id" in user)) throw new Error(`Could not resolve X
|
|
29722
|
+
if (!user || typeof user !== "object" || !("id" in user)) throw new Error(`Could not resolve X username ${input}.`);
|
|
29327
29723
|
return {
|
|
29328
29724
|
id: String(user.id),
|
|
29329
|
-
url: `https://x.com/${"username" in user && typeof user.username === "string" ? user.username :
|
|
29725
|
+
url: `https://x.com/${"username" in user && typeof user.username === "string" ? user.username : inputUsername}`
|
|
29330
29726
|
};
|
|
29331
29727
|
};
|
|
29332
29728
|
const lookupLinkedinProfile = async (input, account, selectedAccount) => {
|
|
@@ -29417,7 +29813,7 @@ const createCLIDeps = () => {
|
|
|
29417
29813
|
return {
|
|
29418
29814
|
account: {
|
|
29419
29815
|
platform: "x",
|
|
29420
|
-
|
|
29816
|
+
username: selected.username,
|
|
29421
29817
|
selector: accountSelectorFor(selected)
|
|
29422
29818
|
},
|
|
29423
29819
|
accountIdHeader: accountSelectorFor(selected),
|
|
@@ -29434,13 +29830,13 @@ const createCLIDeps = () => {
|
|
|
29434
29830
|
if (!selected) throw account ? new NotFoundError("linkedin_account_not_found") : /* @__PURE__ */ new Error("linkedin_account_not_connected");
|
|
29435
29831
|
const profile = await lookupLinkedinProfile("me", account, {
|
|
29436
29832
|
platform: "linkedin",
|
|
29437
|
-
|
|
29833
|
+
username: selected.username,
|
|
29438
29834
|
selector: accountSelectorFor(selected)
|
|
29439
29835
|
});
|
|
29440
29836
|
return {
|
|
29441
29837
|
account: {
|
|
29442
29838
|
platform: "linkedin",
|
|
29443
|
-
|
|
29839
|
+
username: selected.username,
|
|
29444
29840
|
selector: accountSelectorFor(selected)
|
|
29445
29841
|
},
|
|
29446
29842
|
accountIdHeader: accountSelectorFor(selected),
|
|
@@ -29647,7 +30043,6 @@ const compactSchemaForCommand = (command, path) => {
|
|
|
29647
30043
|
const label = commandLabel(command, path);
|
|
29648
30044
|
const { method, capability, mutates } = declaredMetadataFor(command, contractFor(command, label), label, { require: false });
|
|
29649
30045
|
return {
|
|
29650
|
-
schemaVersion: "1.0",
|
|
29651
30046
|
name: path.join(".") || commandName(command, "social"),
|
|
29652
30047
|
description: commandDescription(command),
|
|
29653
30048
|
...method ? { method } : {},
|
|
@@ -29667,7 +30062,6 @@ const compactLeafSchemaForCommand = (command, path) => {
|
|
|
29667
30062
|
if (capability === void 0 || mutates === void 0) throw new Error(`Missing CLI contract metadata for ${label}.`);
|
|
29668
30063
|
const pagination = metadataValueFor(contract.pagination ?? command.meta?.pagination, `${label}.pagination`);
|
|
29669
30064
|
return {
|
|
29670
|
-
schemaVersion: "1.0",
|
|
29671
30065
|
path: path.join(" "),
|
|
29672
30066
|
name: path.join(".") || commandName(command, "social"),
|
|
29673
30067
|
description: commandDescription(command),
|
|
@@ -29694,14 +30088,12 @@ const schemaForCommand = (command, path) => {
|
|
|
29694
30088
|
const pagination = metadataValueFor(contract.pagination ?? command.meta?.pagination, `${label}.pagination`);
|
|
29695
30089
|
if (pagination === void 0) throw new Error(`Missing CLI contract pagination for ${label}.`);
|
|
29696
30090
|
return {
|
|
29697
|
-
schemaVersion: "1.0",
|
|
29698
30091
|
name: path.join(".") || commandName(command, "social"),
|
|
29699
30092
|
description: commandDescription(command),
|
|
29700
30093
|
...method ? { method } : {},
|
|
29701
30094
|
...inputSourceFor(command),
|
|
29702
30095
|
args: metadataValueFor(contract.args, `${label}.args`) ?? positionalArgs,
|
|
29703
30096
|
flags: metadataValueFor(contract.flags, `${label}.flags`) ?? flags,
|
|
29704
|
-
params: metadataValueFor(contract.params, `${label}.params`) ?? args,
|
|
29705
30097
|
inputSchema: inputSchemaFor(command, contract, label, args),
|
|
29706
30098
|
outputSchema,
|
|
29707
30099
|
auth: authFor(contract.auth ?? command.meta?.auth, capability),
|
|
@@ -29719,9 +30111,11 @@ const schemaForCommand = (command, path) => {
|
|
|
29719
30111
|
};
|
|
29720
30112
|
const treeForCommand = (command, path = []) => {
|
|
29721
30113
|
const subCommands = childCommandsFor(command);
|
|
30114
|
+
const entries = Object.entries(subCommands).filter(([, subCommand]) => !isHiddenCommand(subCommand));
|
|
30115
|
+
const visibleSubCommands = Object.fromEntries(entries.map(([name, subCommand]) => [name, treeForCommand(subCommand, [...path, commandName(subCommand, name)])]));
|
|
29722
30116
|
return {
|
|
29723
30117
|
...compactSchemaForCommand(command, path),
|
|
29724
|
-
|
|
30118
|
+
...entries.length > 0 ? { subCommands: visibleSubCommands } : {}
|
|
29725
30119
|
};
|
|
29726
30120
|
};
|
|
29727
30121
|
const leafSchemasForCommand = (command, path = []) => {
|
|
@@ -29807,11 +30201,20 @@ const schemaCommand = defineCommand({
|
|
|
29807
30201
|
return;
|
|
29808
30202
|
}
|
|
29809
30203
|
if (parts.length === 0) {
|
|
29810
|
-
printData(
|
|
30204
|
+
printData({
|
|
30205
|
+
schemaVersion: "1.0",
|
|
30206
|
+
...treeForCommand(rootCommand)
|
|
30207
|
+
});
|
|
29811
30208
|
return;
|
|
29812
30209
|
}
|
|
29813
30210
|
const { command, path } = commandAtPath(rootCommand, parts);
|
|
29814
|
-
printData(command.subCommands && typeof command.run !== "function" ?
|
|
30211
|
+
printData(command.subCommands && typeof command.run !== "function" ? {
|
|
30212
|
+
schemaVersion: "1.0",
|
|
30213
|
+
...treeForCommand(command, path)
|
|
30214
|
+
} : {
|
|
30215
|
+
schemaVersion: "1.0",
|
|
30216
|
+
...schemaForCommand(command, path)
|
|
30217
|
+
});
|
|
29815
30218
|
}
|
|
29816
30219
|
});
|
|
29817
30220
|
//#endregion
|
|
@@ -29918,7 +30321,7 @@ const unknownSubCommandFor = async (command, rawArgs) => {
|
|
|
29918
30321
|
* `commandForUsage` walk but collects names.
|
|
29919
30322
|
*
|
|
29920
30323
|
* PRIVACY: only tokens that resolve to a real subcommand are appended. The first
|
|
29921
|
-
* arg that does not resolve (a positional value: tweet ID,
|
|
30324
|
+
* arg that does not resolve (a positional value: tweet ID, username, query text,
|
|
29922
30325
|
* flag value) breaks the walk and is NEVER captured. Never returns rawArgs or
|
|
29923
30326
|
* flag values.
|
|
29924
30327
|
*/
|