birdclaw 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -0
- package/README.md +55 -5
- package/bin/birdclaw.mjs +50 -11
- package/package.json +9 -7
- package/public/birdclaw-mark.png +0 -0
- package/scripts/browser-perf.mjs +400 -0
- package/scripts/build-docs-site.mjs +940 -0
- package/scripts/docs-site-assets.mjs +311 -0
- package/scripts/run-vitest.mjs +21 -0
- package/scripts/sanitize-node-options.mjs +23 -0
- package/scripts/start-test-server.mjs +42 -0
- package/src/cli.ts +422 -14
- package/src/components/AccountSwitcher.tsx +186 -0
- package/src/components/AppNav.tsx +29 -9
- package/src/components/AvatarChip.tsx +9 -3
- package/src/components/BrandMark.tsx +67 -0
- package/src/components/ConversationThread.tsx +11 -10
- package/src/components/DmWorkspace.tsx +39 -14
- package/src/components/FeedState.tsx +147 -0
- package/src/components/InboxCard.tsx +14 -2
- package/src/components/LinkPreviewCard.tsx +40 -18
- package/src/components/MarkdownViewer.tsx +452 -0
- package/src/components/SavedTimelineView.tsx +64 -56
- package/src/components/SyncNowButton.tsx +137 -0
- package/src/components/ThemeSlider.tsx +49 -93
- package/src/components/TimelineCard.tsx +364 -136
- package/src/components/TimelineRouteFrame.tsx +170 -0
- package/src/components/TweetMediaGrid.tsx +170 -24
- package/src/components/TweetRichText.tsx +28 -13
- package/src/components/account-selection.ts +64 -0
- package/src/components/useTimelineRouteData.ts +153 -0
- package/src/lib/account-sync-job.ts +654 -0
- package/src/lib/actions-transport.ts +216 -146
- package/src/lib/api-client.ts +304 -0
- package/src/lib/archive-finder.ts +72 -53
- package/src/lib/archive-import.ts +1377 -1298
- package/src/lib/authored-live.ts +261 -204
- package/src/lib/avatar-cache.ts +159 -44
- package/src/lib/backup.ts +1532 -951
- package/src/lib/bird-actions.ts +139 -57
- package/src/lib/bird-command.ts +101 -28
- package/src/lib/bird.ts +549 -194
- package/src/lib/blocklist.ts +40 -23
- package/src/lib/blocks-write.ts +129 -80
- package/src/lib/blocks.ts +165 -97
- package/src/lib/bookmark-sync-job.ts +250 -160
- package/src/lib/conversation-surface.ts +205 -0
- package/src/lib/db.ts +35 -3
- package/src/lib/dms-live.ts +720 -66
- package/src/lib/effect-runtime.ts +45 -0
- package/src/lib/follow-graph.ts +224 -180
- package/src/lib/http-effect.ts +222 -0
- package/src/lib/inbox.ts +74 -43
- package/src/lib/link-index.ts +88 -76
- package/src/lib/link-insights.ts +24 -0
- package/src/lib/link-preview-metadata.ts +472 -52
- package/src/lib/media-fetch.ts +286 -213
- package/src/lib/mention-threads-live.ts +352 -288
- package/src/lib/mentions-live.ts +390 -342
- package/src/lib/moderation-target.ts +102 -65
- package/src/lib/moderation-write.ts +77 -18
- package/src/lib/mutes-write.ts +129 -80
- package/src/lib/mutes.ts +8 -1
- package/src/lib/openai.ts +84 -53
- package/src/lib/period-digest.ts +953 -0
- package/src/lib/profile-affiliation-hydration.ts +93 -54
- package/src/lib/profile-hydration.ts +124 -72
- package/src/lib/profile-replies.ts +60 -43
- package/src/lib/profile-resolver.ts +402 -294
- package/src/lib/queries.ts +1024 -189
- package/src/lib/research.ts +165 -120
- package/src/lib/sqlite.ts +1 -0
- package/src/lib/timeline-collections-live.ts +204 -167
- package/src/lib/timeline-live.ts +60 -39
- package/src/lib/tweet-lookup.ts +30 -19
- package/src/lib/types.ts +38 -1
- package/src/lib/ui.ts +41 -10
- package/src/lib/url-expansion.ts +226 -55
- package/src/lib/url-safety.ts +220 -0
- package/src/lib/web-sync.ts +511 -0
- package/src/lib/whois.ts +166 -147
- package/src/lib/x-web.ts +102 -71
- package/src/lib/xurl.ts +681 -411
- package/src/routeTree.gen.ts +63 -0
- package/src/routes/__root.tsx +25 -5
- package/src/routes/api/action.tsx +127 -78
- package/src/routes/api/avatar.tsx +39 -30
- package/src/routes/api/blocks.tsx +26 -23
- package/src/routes/api/conversation.tsx +25 -14
- package/src/routes/api/inbox.tsx +27 -21
- package/src/routes/api/link-insights.tsx +31 -25
- package/src/routes/api/link-preview.tsx +25 -21
- package/src/routes/api/period-digest.tsx +123 -0
- package/src/routes/api/profile-hydrate.tsx +31 -28
- package/src/routes/api/query.tsx +79 -55
- package/src/routes/api/status.tsx +18 -10
- package/src/routes/api/sync.tsx +105 -0
- package/src/routes/dms.tsx +195 -55
- package/src/routes/inbox.tsx +32 -19
- package/src/routes/index.tsx +21 -127
- package/src/routes/links.tsx +50 -5
- package/src/routes/mentions.tsx +21 -127
- package/src/routes/today.tsx +441 -0
- package/src/styles.css +74 -11
- package/vite.config.ts +8 -0
|
@@ -11,8 +11,11 @@ import {
|
|
|
11
11
|
import path from "node:path";
|
|
12
12
|
import { pipeline } from "node:stream/promises";
|
|
13
13
|
import { promisify } from "node:util";
|
|
14
|
+
import { Effect } from "effect";
|
|
14
15
|
import { getBirdclawPaths } from "./config";
|
|
15
16
|
import { getNativeDb } from "./db";
|
|
17
|
+
import { runEffectPromise, tryPromise } from "./effect-runtime";
|
|
18
|
+
import { safeHttpUrl } from "./url-safety";
|
|
16
19
|
|
|
17
20
|
const execFileAsync = promisify(execFile);
|
|
18
21
|
const ARCHIVE_JSON_PAYLOAD = /=\s*(\[[\s\S]*\]|\{[\s\S]*\})/s;
|
|
@@ -108,48 +111,60 @@ function parseArchiveArray(content: string): ArchiveRecord[] {
|
|
|
108
111
|
: [];
|
|
109
112
|
}
|
|
110
113
|
|
|
111
|
-
|
|
114
|
+
function runUnzipEffect(
|
|
112
115
|
_archivePath: string,
|
|
113
116
|
args: string[],
|
|
114
117
|
maxBuffer = 1024 * 1024 * 256,
|
|
115
118
|
) {
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
|
|
119
|
+
return tryPromise(() =>
|
|
120
|
+
execFileAsync("unzip", args, {
|
|
121
|
+
maxBuffer,
|
|
122
|
+
}),
|
|
123
|
+
).pipe(Effect.map(({ stdout }) => stdout));
|
|
120
124
|
}
|
|
121
125
|
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
126
|
+
function listArchiveEntriesEffect(
|
|
127
|
+
archivePath: string,
|
|
128
|
+
): Effect.Effect<string[], unknown> {
|
|
129
|
+
return Effect.gen(function* () {
|
|
130
|
+
const stdout = yield* runUnzipEffect(
|
|
131
|
+
archivePath,
|
|
132
|
+
["-Z1", archivePath],
|
|
133
|
+
1024 * 1024 * 64,
|
|
134
|
+
);
|
|
135
|
+
return stdout
|
|
136
|
+
.split("\n")
|
|
137
|
+
.map((item) => item.trim())
|
|
138
|
+
.filter((item) => item.length > 0);
|
|
139
|
+
});
|
|
132
140
|
}
|
|
133
141
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
142
|
+
function listArchiveEntryDetailsEffect(
|
|
143
|
+
archivePath: string,
|
|
144
|
+
): Effect.Effect<Array<{ path: string; size: number }>, unknown> {
|
|
145
|
+
return Effect.gen(function* () {
|
|
146
|
+
const stdout = yield* runUnzipEffect(
|
|
147
|
+
archivePath,
|
|
148
|
+
["-Z", "-l", archivePath],
|
|
149
|
+
1024 * 1024 * 64,
|
|
150
|
+
);
|
|
151
|
+
return stdout
|
|
152
|
+
.split("\n")
|
|
153
|
+
.map((line) => line.trim().split(/\s+/))
|
|
154
|
+
.filter((parts) => parts.length >= 10 && /^[-d]/.test(parts[0] ?? ""))
|
|
155
|
+
.map((parts) => ({
|
|
156
|
+
path: parts.slice(9).join(" "),
|
|
157
|
+
size: Number(parts[3] ?? 0),
|
|
158
|
+
}))
|
|
159
|
+
.filter((entry) => entry.path.length > 0 && Number.isFinite(entry.size));
|
|
160
|
+
});
|
|
149
161
|
}
|
|
150
162
|
|
|
151
|
-
|
|
152
|
-
|
|
163
|
+
function readArchiveEntryEffect(
|
|
164
|
+
archivePath: string,
|
|
165
|
+
entryPath: string,
|
|
166
|
+
): Effect.Effect<string, unknown> {
|
|
167
|
+
return runUnzipEffect(archivePath, ["-p", archivePath, entryPath]);
|
|
153
168
|
}
|
|
154
169
|
|
|
155
170
|
function getFirstEntry(entries: string[], pattern: RegExp) {
|
|
@@ -216,14 +231,23 @@ function toFiniteNumber(value: unknown) {
|
|
|
216
231
|
return Number.isFinite(number) ? number : undefined;
|
|
217
232
|
}
|
|
218
233
|
|
|
234
|
+
function archiveHttpUrl(value: unknown) {
|
|
235
|
+
return safeHttpUrl(typeof value === "string" ? value : String(value ?? ""));
|
|
236
|
+
}
|
|
237
|
+
|
|
219
238
|
function extractTweetEntities(tweet: Record<string, unknown>) {
|
|
220
239
|
const entities = asRecord(tweet.entities);
|
|
221
|
-
const
|
|
240
|
+
const urlEntries = [
|
|
241
|
+
...asArray<Record<string, unknown>>(entities?.urls),
|
|
242
|
+
...asArray<Record<string, unknown>>(entities?.media),
|
|
243
|
+
];
|
|
244
|
+
const seenUrls = new Set<string>();
|
|
245
|
+
const urls = urlEntries
|
|
222
246
|
.map((entry) => ({
|
|
223
|
-
url:
|
|
224
|
-
expandedUrl:
|
|
225
|
-
entry.expanded_url ?? entry.expandedUrl ?? entry.url ??
|
|
226
|
-
|
|
247
|
+
url: archiveHttpUrl(entry.url) ?? "",
|
|
248
|
+
expandedUrl:
|
|
249
|
+
archiveHttpUrl(entry.expanded_url ?? entry.expandedUrl ?? entry.url) ??
|
|
250
|
+
"",
|
|
227
251
|
displayUrl: String(
|
|
228
252
|
entry.display_url ??
|
|
229
253
|
entry.displayUrl ??
|
|
@@ -237,13 +261,13 @@ function extractTweetEntities(tweet: Record<string, unknown>) {
|
|
|
237
261
|
description:
|
|
238
262
|
typeof entry.description === "string" ? entry.description : null,
|
|
239
263
|
imageUrl:
|
|
240
|
-
|
|
241
|
-
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
264
|
+
archiveHttpUrl(
|
|
265
|
+
entry.image_url ??
|
|
266
|
+
entry.imageUrl ??
|
|
267
|
+
entry.thumbnail_url ??
|
|
268
|
+
entry.media_url_https ??
|
|
269
|
+
entry.media_url,
|
|
270
|
+
) ?? undefined,
|
|
247
271
|
siteName:
|
|
248
272
|
typeof entry.site_name === "string"
|
|
249
273
|
? entry.site_name
|
|
@@ -251,7 +275,13 @@ function extractTweetEntities(tweet: Record<string, unknown>) {
|
|
|
251
275
|
? entry.siteName
|
|
252
276
|
: undefined,
|
|
253
277
|
}))
|
|
254
|
-
.filter((entry) => entry.url.length > 0 || entry.expandedUrl.length > 0)
|
|
278
|
+
.filter((entry) => entry.url.length > 0 || entry.expandedUrl.length > 0)
|
|
279
|
+
.filter((entry) => {
|
|
280
|
+
const key = `${entry.start}:${entry.end}:${entry.url}:${entry.expandedUrl}`;
|
|
281
|
+
if (seenUrls.has(key)) return false;
|
|
282
|
+
seenUrls.add(key);
|
|
283
|
+
return true;
|
|
284
|
+
});
|
|
255
285
|
const mentions = asArray<Record<string, unknown>>(entities?.user_mentions)
|
|
256
286
|
.map((entry) => ({
|
|
257
287
|
username: String(entry.screen_name ?? ""),
|
|
@@ -341,12 +371,11 @@ function extractTweetMedia(tweet: Record<string, unknown>) {
|
|
|
341
371
|
|
|
342
372
|
return sourceMedia
|
|
343
373
|
.map((entry) => {
|
|
344
|
-
const url =
|
|
345
|
-
entry.media_url_https ?? entry.media_url ?? entry.url ??
|
|
346
|
-
|
|
347
|
-
const thumbnailUrl =
|
|
348
|
-
entry.media_url_https ?? entry.media_url ?? url
|
|
349
|
-
);
|
|
374
|
+
const url =
|
|
375
|
+
archiveHttpUrl(entry.media_url_https ?? entry.media_url ?? entry.url) ??
|
|
376
|
+
"";
|
|
377
|
+
const thumbnailUrl =
|
|
378
|
+
archiveHttpUrl(entry.media_url_https ?? entry.media_url ?? url) ?? url;
|
|
350
379
|
const videoInfo = asRecord(entry.video_info);
|
|
351
380
|
const durationMs = toFiniteNumber(videoInfo?.duration_millis);
|
|
352
381
|
const variants = archiveMp4Variants(entry);
|
|
@@ -500,66 +529,78 @@ function needsArchiveMediaCopy(destinationPath: string, size: number) {
|
|
|
500
529
|
return statSync(destinationPath).size !== size;
|
|
501
530
|
}
|
|
502
531
|
|
|
503
|
-
|
|
532
|
+
function copyArchiveEntryToFileEffect(
|
|
504
533
|
archivePath: string,
|
|
505
534
|
entryPath: string,
|
|
506
535
|
destinationPath: string,
|
|
507
536
|
) {
|
|
508
|
-
|
|
509
|
-
|
|
510
|
-
|
|
511
|
-
|
|
512
|
-
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
stderr
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
537
|
+
return tryPromise(() => {
|
|
538
|
+
mkdirSync(path.dirname(destinationPath), { recursive: true });
|
|
539
|
+
const temporaryPath = `${destinationPath}.${process.pid}.${randomUUID()}.tmp`;
|
|
540
|
+
const child = spawn("unzip", ["-p", archivePath, entryPath], {
|
|
541
|
+
stdio: ["ignore", "pipe", "pipe"],
|
|
542
|
+
});
|
|
543
|
+
let stderr = "";
|
|
544
|
+
child.stderr.setEncoding("utf8");
|
|
545
|
+
child.stderr.on("data", (chunk) => {
|
|
546
|
+
stderr += String(chunk);
|
|
547
|
+
});
|
|
548
|
+
const exit = new Promise<number | null>((resolve, reject) => {
|
|
549
|
+
child.on("error", reject);
|
|
550
|
+
child.on("close", resolve);
|
|
551
|
+
});
|
|
522
552
|
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
528
|
-
|
|
529
|
-
|
|
530
|
-
|
|
531
|
-
|
|
532
|
-
|
|
533
|
-
|
|
534
|
-
|
|
535
|
-
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
553
|
+
return pipeline(child.stdout, createWriteStream(temporaryPath))
|
|
554
|
+
.then(() => exit)
|
|
555
|
+
.then((exitCode) => {
|
|
556
|
+
if (exitCode !== 0) {
|
|
557
|
+
throw new Error(
|
|
558
|
+
`Failed to extract ${entryPath}: ${
|
|
559
|
+
stderr.trim() || `exit ${String(exitCode)}`
|
|
560
|
+
}`,
|
|
561
|
+
);
|
|
562
|
+
}
|
|
563
|
+
renameSync(temporaryPath, destinationPath);
|
|
564
|
+
})
|
|
565
|
+
.catch((error: unknown) => {
|
|
566
|
+
child.kill();
|
|
567
|
+
if (existsSync(temporaryPath)) {
|
|
568
|
+
unlinkSync(temporaryPath);
|
|
569
|
+
}
|
|
570
|
+
throw error;
|
|
571
|
+
});
|
|
572
|
+
});
|
|
539
573
|
}
|
|
540
574
|
|
|
541
|
-
|
|
575
|
+
function extractArchiveMediaFilesEffect(
|
|
542
576
|
archivePath: string,
|
|
543
577
|
selectedKinds: Set<ArchiveMediaKind> | null,
|
|
544
|
-
) {
|
|
545
|
-
|
|
546
|
-
|
|
578
|
+
): Effect.Effect<ArchiveMediaFileCounts, unknown> {
|
|
579
|
+
return Effect.gen(function* () {
|
|
580
|
+
const counts = createArchiveMediaFileCounts();
|
|
581
|
+
if (selectedKinds?.size === 0) {
|
|
582
|
+
return counts;
|
|
583
|
+
}
|
|
584
|
+
const entries = yield* listArchiveEntryDetailsEffect(archivePath);
|
|
585
|
+
for (const entry of entries) {
|
|
586
|
+
const mediaKind = getArchiveMediaKind(entry.path);
|
|
587
|
+
if (!mediaKind) continue;
|
|
588
|
+
if (selectedKinds && !selectedKinds.has(mediaKind.kind)) continue;
|
|
589
|
+
|
|
590
|
+
counts[mediaKind.kind] += 1;
|
|
591
|
+
const destinationPath = getArchiveMediaDestination(
|
|
592
|
+
entry.path,
|
|
593
|
+
mediaKind.kind,
|
|
594
|
+
);
|
|
595
|
+
if (!needsArchiveMediaCopy(destinationPath, entry.size)) continue;
|
|
596
|
+
yield* copyArchiveEntryToFileEffect(
|
|
597
|
+
archivePath,
|
|
598
|
+
entry.path,
|
|
599
|
+
destinationPath,
|
|
600
|
+
);
|
|
601
|
+
}
|
|
547
602
|
return counts;
|
|
548
|
-
}
|
|
549
|
-
for (const entry of await listArchiveEntryDetails(archivePath)) {
|
|
550
|
-
const mediaKind = getArchiveMediaKind(entry.path);
|
|
551
|
-
if (!mediaKind) continue;
|
|
552
|
-
if (selectedKinds && !selectedKinds.has(mediaKind.kind)) continue;
|
|
553
|
-
|
|
554
|
-
counts[mediaKind.kind] += 1;
|
|
555
|
-
const destinationPath = getArchiveMediaDestination(
|
|
556
|
-
entry.path,
|
|
557
|
-
mediaKind.kind,
|
|
558
|
-
);
|
|
559
|
-
if (!needsArchiveMediaCopy(destinationPath, entry.size)) continue;
|
|
560
|
-
await copyArchiveEntryToFile(archivePath, entry.path, destinationPath);
|
|
561
|
-
}
|
|
562
|
-
return counts;
|
|
603
|
+
});
|
|
563
604
|
}
|
|
564
605
|
|
|
565
606
|
function getArchiveFollowRows(content: string, key: ArchiveFollowKey) {
|
|
@@ -579,8 +620,7 @@ function getArchiveFollowRows(content: string, key: ArchiveFollowKey) {
|
|
|
579
620
|
return rows;
|
|
580
621
|
}
|
|
581
622
|
|
|
582
|
-
function clearImportedData() {
|
|
583
|
-
const db = getNativeDb();
|
|
623
|
+
function clearImportedData(db = getNativeDb()) {
|
|
584
624
|
db.exec(`
|
|
585
625
|
delete from ai_scores;
|
|
586
626
|
delete from tweet_actions;
|
|
@@ -617,894 +657,914 @@ function clearMentionSyncState(db = getNativeDb()) {
|
|
|
617
657
|
).run();
|
|
618
658
|
}
|
|
619
659
|
|
|
620
|
-
|
|
660
|
+
function importArchiveInternalEffect(
|
|
621
661
|
archivePath: string,
|
|
622
662
|
options: ImportArchiveOptions = {},
|
|
623
|
-
):
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
636
|
-
|
|
637
|
-
|
|
638
|
-
|
|
639
|
-
|
|
640
|
-
|
|
641
|
-
|
|
642
|
-
|
|
643
|
-
|
|
644
|
-
|
|
645
|
-
|
|
646
|
-
|
|
647
|
-
|
|
648
|
-
|
|
649
|
-
|
|
650
|
-
|
|
651
|
-
|
|
652
|
-
|
|
653
|
-
|
|
654
|
-
|
|
655
|
-
|
|
656
|
-
|
|
657
|
-
|
|
658
|
-
|
|
659
|
-
|
|
660
|
-
|
|
661
|
-
|
|
662
|
-
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
|
|
663
|
+
): Effect.Effect<ImportedArchiveSummary, unknown> {
|
|
664
|
+
return Effect.gen(function* () {
|
|
665
|
+
const entries = yield* listArchiveEntriesEffect(archivePath);
|
|
666
|
+
const selection = selectedSlices(options);
|
|
667
|
+
const includeTweets = includesSlice(selection, "tweets");
|
|
668
|
+
const includeLikes = includesSlice(selection, "likes");
|
|
669
|
+
const includeBookmarks = includesSlice(selection, "bookmarks");
|
|
670
|
+
const includeDirectMessages = includesSlice(selection, "directMessages");
|
|
671
|
+
const includeProfiles = includesSlice(selection, "profiles");
|
|
672
|
+
const includeFollowers = includesSlice(selection, "followers");
|
|
673
|
+
const includeFollowing = includesSlice(selection, "following");
|
|
674
|
+
const accountEntry = getFirstEntry(entries, /(?:^|\/)data\/account\.js$/i);
|
|
675
|
+
const profileEntry = getFirstEntry(entries, /(?:^|\/)data\/profile\.js$/i);
|
|
676
|
+
const tweetEntries = includeTweets
|
|
677
|
+
? getMatchingEntries(
|
|
678
|
+
entries,
|
|
679
|
+
/(?:^|\/)data\/(?:tweets|community-tweet)(?:-part\d+)?\.js$/i,
|
|
680
|
+
)
|
|
681
|
+
: [];
|
|
682
|
+
const noteTweetEntries = includeTweets
|
|
683
|
+
? getMatchingEntries(
|
|
684
|
+
entries,
|
|
685
|
+
/(?:^|\/)data\/note-tweet(?:-part\d+)?\.js$/i,
|
|
686
|
+
)
|
|
687
|
+
: [];
|
|
688
|
+
const likeEntries = includeLikes
|
|
689
|
+
? getMatchingEntries(
|
|
690
|
+
entries,
|
|
691
|
+
/(?:^|\/)data\/(?:like|likes)(?:-part\d+)?\.js$/i,
|
|
692
|
+
)
|
|
693
|
+
: [];
|
|
694
|
+
const bookmarkEntries = includeBookmarks
|
|
695
|
+
? getMatchingEntries(
|
|
696
|
+
entries,
|
|
697
|
+
/(?:^|\/)data\/(?:bookmark|bookmarks)(?:-part\d+)?\.js$/i,
|
|
698
|
+
)
|
|
699
|
+
: [];
|
|
700
|
+
const dmEntries = includeDirectMessages
|
|
701
|
+
? getMatchingEntries(
|
|
702
|
+
entries,
|
|
703
|
+
/(?:^|\/)data\/direct-messages(?:-group)?(?:-part\d+)?\.js$/i,
|
|
704
|
+
)
|
|
705
|
+
: [];
|
|
706
|
+
const followerEntries = includeFollowers
|
|
707
|
+
? getMatchingEntries(entries, /(?:^|\/)data\/follower(?:-part\d+)?\.js$/i)
|
|
708
|
+
: [];
|
|
709
|
+
const followingEntries = includeFollowing
|
|
710
|
+
? getMatchingEntries(
|
|
711
|
+
entries,
|
|
712
|
+
/(?:^|\/)data\/following(?:-part\d+)?\.js$/i,
|
|
713
|
+
)
|
|
714
|
+
: [];
|
|
668
715
|
|
|
669
|
-
|
|
670
|
-
|
|
671
|
-
|
|
716
|
+
if (!accountEntry) {
|
|
717
|
+
return yield* Effect.fail(new Error("Archive missing data/account.js"));
|
|
718
|
+
}
|
|
672
719
|
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
|
|
720
|
+
const [accountContent, profileContent] = yield* Effect.all([
|
|
721
|
+
readArchiveEntryEffect(archivePath, accountEntry),
|
|
722
|
+
profileEntry
|
|
723
|
+
? readArchiveEntryEffect(archivePath, profileEntry)
|
|
724
|
+
: Effect.succeed("[]"),
|
|
725
|
+
]);
|
|
679
726
|
|
|
680
|
-
|
|
681
|
-
|
|
682
|
-
|
|
683
|
-
|
|
727
|
+
const accountPayload = buildAccountPayload(
|
|
728
|
+
parseArchiveArray(accountContent)[0] ?? null,
|
|
729
|
+
parseArchiveArray(profileContent)[0] ?? null,
|
|
730
|
+
);
|
|
684
731
|
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
706
|
-
|
|
707
|
-
|
|
708
|
-
|
|
709
|
-
|
|
710
|
-
|
|
711
|
-
|
|
712
|
-
|
|
713
|
-
|
|
714
|
-
|
|
715
|
-
|
|
716
|
-
|
|
717
|
-
|
|
718
|
-
|
|
719
|
-
|
|
720
|
-
|
|
732
|
+
const mentionDirectory = new Map<
|
|
733
|
+
string,
|
|
734
|
+
{ handle?: string; displayName?: string }
|
|
735
|
+
>();
|
|
736
|
+
const tweetRows: Array<{
|
|
737
|
+
id: string;
|
|
738
|
+
kind: "home" | "like" | "bookmark";
|
|
739
|
+
authorProfileId: string;
|
|
740
|
+
text: string;
|
|
741
|
+
createdAt: string;
|
|
742
|
+
isReplied: number;
|
|
743
|
+
replyToId: string | null;
|
|
744
|
+
likeCount: number;
|
|
745
|
+
mediaCount: number;
|
|
746
|
+
bookmarked: number;
|
|
747
|
+
liked: number;
|
|
748
|
+
entitiesJson: string;
|
|
749
|
+
mediaJson: string;
|
|
750
|
+
quotedTweetId: string | null;
|
|
751
|
+
}> = [];
|
|
752
|
+
const collectionRows: Array<{
|
|
753
|
+
tweetId: string;
|
|
754
|
+
kind: "likes" | "bookmarks";
|
|
755
|
+
collectedAt: string | null;
|
|
756
|
+
source: string;
|
|
757
|
+
rawJson: string;
|
|
758
|
+
}> = [];
|
|
759
|
+
const tweetRowsById = new Map<string, (typeof tweetRows)[number]>();
|
|
760
|
+
|
|
761
|
+
function addTweetRow(row: (typeof tweetRows)[number]) {
|
|
762
|
+
const existing = tweetRowsById.get(row.id);
|
|
763
|
+
if (existing) {
|
|
764
|
+
existing.bookmarked = Math.max(existing.bookmarked, row.bookmarked);
|
|
765
|
+
existing.liked = Math.max(existing.liked, row.liked);
|
|
766
|
+
if (!existing.text && row.text) existing.text = row.text;
|
|
767
|
+
return;
|
|
768
|
+
}
|
|
769
|
+
tweetRows.push(row);
|
|
770
|
+
tweetRowsById.set(row.id, row);
|
|
721
771
|
}
|
|
722
|
-
tweetRows.push(row);
|
|
723
|
-
tweetRowsById.set(row.id, row);
|
|
724
|
-
}
|
|
725
772
|
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
773
|
+
for (const entry of tweetEntries) {
|
|
774
|
+
const content = yield* readArchiveEntryEffect(archivePath, entry);
|
|
775
|
+
for (const wrapper of parseArchiveArray(content)) {
|
|
776
|
+
const tweet = asRecord(wrapper.tweet);
|
|
777
|
+
if (!tweet) continue;
|
|
778
|
+
|
|
779
|
+
for (const mention of asArray<Record<string, unknown>>(
|
|
780
|
+
asRecord(tweet.entities)?.user_mentions,
|
|
781
|
+
)) {
|
|
782
|
+
const mentionId = String(mention.id_str ?? mention.id ?? "");
|
|
783
|
+
if (!mentionId) continue;
|
|
784
|
+
mentionDirectory.set(mentionId, {
|
|
785
|
+
handle: String(mention.screen_name ?? ""),
|
|
786
|
+
displayName: String(
|
|
787
|
+
mention.name ?? mention.screen_name ?? mentionId,
|
|
788
|
+
),
|
|
789
|
+
});
|
|
790
|
+
}
|
|
742
791
|
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
792
|
+
const replyUserId = String(
|
|
793
|
+
tweet.in_reply_to_user_id_str ?? tweet.in_reply_to_user_id ?? "",
|
|
794
|
+
);
|
|
795
|
+
const replyScreenName = String(tweet.in_reply_to_screen_name ?? "");
|
|
796
|
+
if (replyUserId && replyScreenName) {
|
|
797
|
+
mentionDirectory.set(replyUserId, {
|
|
798
|
+
handle: replyScreenName,
|
|
799
|
+
displayName: replyScreenName,
|
|
800
|
+
});
|
|
801
|
+
}
|
|
802
|
+
|
|
803
|
+
addTweetRow({
|
|
804
|
+
id: String(tweet.id_str ?? tweet.id),
|
|
805
|
+
kind: "home",
|
|
806
|
+
authorProfileId: "profile_me",
|
|
807
|
+
text: String(tweet.full_text ?? tweet.text ?? ""),
|
|
808
|
+
createdAt: parseTwitterDate(tweet.created_at),
|
|
809
|
+
isReplied: tweet.in_reply_to_status_id_str ? 1 : 0,
|
|
810
|
+
replyToId: tweet.in_reply_to_status_id_str
|
|
811
|
+
? String(tweet.in_reply_to_status_id_str)
|
|
812
|
+
: null,
|
|
813
|
+
likeCount: toInt(tweet.favorite_count),
|
|
814
|
+
mediaCount: getTweetMediaCount(tweet),
|
|
815
|
+
bookmarked: 0,
|
|
816
|
+
liked: 0,
|
|
817
|
+
entitiesJson: JSON.stringify(extractTweetEntities(tweet)),
|
|
818
|
+
mediaJson: JSON.stringify(extractTweetMedia(tweet)),
|
|
819
|
+
quotedTweetId: tweet.quoted_status_id_str
|
|
820
|
+
? String(tweet.quoted_status_id_str)
|
|
821
|
+
: null,
|
|
751
822
|
});
|
|
752
823
|
}
|
|
753
|
-
|
|
754
|
-
addTweetRow({
|
|
755
|
-
id: String(tweet.id_str ?? tweet.id),
|
|
756
|
-
kind: "home",
|
|
757
|
-
authorProfileId: "profile_me",
|
|
758
|
-
text: String(tweet.full_text ?? tweet.text ?? ""),
|
|
759
|
-
createdAt: parseTwitterDate(tweet.created_at),
|
|
760
|
-
isReplied: tweet.in_reply_to_status_id_str ? 1 : 0,
|
|
761
|
-
replyToId: tweet.in_reply_to_status_id_str
|
|
762
|
-
? String(tweet.in_reply_to_status_id_str)
|
|
763
|
-
: null,
|
|
764
|
-
likeCount: toInt(tweet.favorite_count),
|
|
765
|
-
mediaCount: getTweetMediaCount(tweet),
|
|
766
|
-
bookmarked: 0,
|
|
767
|
-
liked: 0,
|
|
768
|
-
entitiesJson: JSON.stringify(extractTweetEntities(tweet)),
|
|
769
|
-
mediaJson: JSON.stringify(extractTweetMedia(tweet)),
|
|
770
|
-
quotedTweetId: tweet.quoted_status_id_str
|
|
771
|
-
? String(tweet.quoted_status_id_str)
|
|
772
|
-
: null,
|
|
773
|
-
});
|
|
774
824
|
}
|
|
775
|
-
}
|
|
776
825
|
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
|
|
780
|
-
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
|
|
784
|
-
|
|
785
|
-
|
|
786
|
-
|
|
787
|
-
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
791
|
-
|
|
792
|
-
|
|
793
|
-
|
|
794
|
-
|
|
795
|
-
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
|
|
826
|
+
for (const entry of noteTweetEntries) {
|
|
827
|
+
const content = yield* readArchiveEntryEffect(archivePath, entry);
|
|
828
|
+
for (const wrapper of parseArchiveArray(content)) {
|
|
829
|
+
const noteTweet = asRecord(wrapper.noteTweet);
|
|
830
|
+
if (!noteTweet) continue;
|
|
831
|
+
const core = asRecord(noteTweet.core);
|
|
832
|
+
addTweetRow({
|
|
833
|
+
id: String(noteTweet.noteTweetId ?? noteTweet.id ?? randomUUID()),
|
|
834
|
+
kind: "home",
|
|
835
|
+
authorProfileId: "profile_me",
|
|
836
|
+
text: String(core?.text ?? ""),
|
|
837
|
+
createdAt: parseTwitterDate(noteTweet.createdAt),
|
|
838
|
+
isReplied: 0,
|
|
839
|
+
replyToId: null,
|
|
840
|
+
likeCount: 0,
|
|
841
|
+
mediaCount: 0,
|
|
842
|
+
bookmarked: 0,
|
|
843
|
+
liked: 0,
|
|
844
|
+
entitiesJson: "{}",
|
|
845
|
+
mediaJson: "[]",
|
|
846
|
+
quotedTweetId: null,
|
|
847
|
+
});
|
|
848
|
+
}
|
|
799
849
|
}
|
|
800
|
-
|
|
801
|
-
const authoredTweetCount = tweetRows.length;
|
|
850
|
+
const authoredTweetCount = tweetRows.length;
|
|
802
851
|
|
|
803
|
-
|
|
804
|
-
|
|
805
|
-
|
|
806
|
-
|
|
807
|
-
|
|
808
|
-
|
|
809
|
-
|
|
810
|
-
|
|
811
|
-
|
|
852
|
+
type MessageRow = {
|
|
853
|
+
id: string;
|
|
854
|
+
conversationId: string;
|
|
855
|
+
senderProfileId: string;
|
|
856
|
+
text: string;
|
|
857
|
+
createdAt: string;
|
|
858
|
+
direction: "inbound" | "outbound";
|
|
859
|
+
mediaCount: number;
|
|
860
|
+
};
|
|
812
861
|
|
|
813
|
-
|
|
814
|
-
|
|
815
|
-
|
|
862
|
+
const profiles = new Map<
|
|
863
|
+
string,
|
|
864
|
+
{
|
|
865
|
+
id: string;
|
|
866
|
+
handle: string;
|
|
867
|
+
displayName: string;
|
|
868
|
+
bio: string;
|
|
869
|
+
followersCount: number;
|
|
870
|
+
followingCount: number;
|
|
871
|
+
publicMetricsJson: string;
|
|
872
|
+
avatarHue: number;
|
|
873
|
+
avatarUrl: string | null;
|
|
874
|
+
location: string | null;
|
|
875
|
+
url: string | null;
|
|
876
|
+
verifiedType: string | null;
|
|
877
|
+
entitiesJson: string;
|
|
878
|
+
rawJson: string;
|
|
879
|
+
createdAt: string;
|
|
880
|
+
}
|
|
881
|
+
>();
|
|
882
|
+
type ProfileRow =
|
|
883
|
+
typeof profiles extends Map<string, infer Value> ? Value : never;
|
|
884
|
+
const defaultProfileMetadata = {
|
|
885
|
+
publicMetricsJson: "{}",
|
|
886
|
+
location: null,
|
|
887
|
+
url: null,
|
|
888
|
+
verifiedType: null,
|
|
889
|
+
entitiesJson: "{}",
|
|
890
|
+
rawJson: "{}",
|
|
891
|
+
};
|
|
892
|
+
const conversations = new Map<
|
|
893
|
+
string,
|
|
894
|
+
{
|
|
895
|
+
id: string;
|
|
896
|
+
title: string;
|
|
897
|
+
accountId: string;
|
|
898
|
+
participantProfileId: string;
|
|
899
|
+
lastMessageAt: string;
|
|
900
|
+
unreadCount: number;
|
|
901
|
+
needsReply: number;
|
|
902
|
+
}
|
|
903
|
+
>();
|
|
904
|
+
const dmMessages: MessageRow[] = [];
|
|
905
|
+
const followerRows: Array<{ profileId: string; externalUserId: string }> =
|
|
906
|
+
[];
|
|
907
|
+
const followingRows: Array<{ profileId: string; externalUserId: string }> =
|
|
908
|
+
[];
|
|
909
|
+
const followerIds = new Set<string>();
|
|
910
|
+
const followingIds = new Set<string>();
|
|
911
|
+
type ExistingProfileRow = {
|
|
816
912
|
id: string;
|
|
817
913
|
handle: string;
|
|
818
|
-
|
|
914
|
+
display_name: string;
|
|
819
915
|
bio: string;
|
|
820
|
-
|
|
821
|
-
|
|
822
|
-
|
|
823
|
-
|
|
824
|
-
|
|
916
|
+
followers_count: number;
|
|
917
|
+
following_count: number;
|
|
918
|
+
public_metrics_json: string;
|
|
919
|
+
avatar_hue: number;
|
|
920
|
+
avatar_url: string | null;
|
|
825
921
|
location: string | null;
|
|
826
922
|
url: string | null;
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
|
|
830
|
-
|
|
831
|
-
}
|
|
832
|
-
|
|
833
|
-
|
|
834
|
-
|
|
835
|
-
|
|
836
|
-
|
|
837
|
-
location: null,
|
|
838
|
-
url: null,
|
|
839
|
-
verifiedType: null,
|
|
840
|
-
entitiesJson: "{}",
|
|
841
|
-
rawJson: "{}",
|
|
842
|
-
};
|
|
843
|
-
const conversations = new Map<
|
|
844
|
-
string,
|
|
845
|
-
{
|
|
846
|
-
id: string;
|
|
847
|
-
title: string;
|
|
848
|
-
accountId: string;
|
|
849
|
-
participantProfileId: string;
|
|
850
|
-
lastMessageAt: string;
|
|
851
|
-
unreadCount: number;
|
|
852
|
-
needsReply: number;
|
|
853
|
-
}
|
|
854
|
-
>();
|
|
855
|
-
const dmMessages: MessageRow[] = [];
|
|
856
|
-
const followerRows: Array<{ profileId: string; externalUserId: string }> = [];
|
|
857
|
-
const followingRows: Array<{ profileId: string; externalUserId: string }> =
|
|
858
|
-
[];
|
|
859
|
-
const followerIds = new Set<string>();
|
|
860
|
-
const followingIds = new Set<string>();
|
|
861
|
-
type ExistingProfileRow = {
|
|
862
|
-
id: string;
|
|
863
|
-
handle: string;
|
|
864
|
-
display_name: string;
|
|
865
|
-
bio: string;
|
|
866
|
-
followers_count: number;
|
|
867
|
-
following_count: number;
|
|
868
|
-
public_metrics_json: string;
|
|
869
|
-
avatar_hue: number;
|
|
870
|
-
avatar_url: string | null;
|
|
871
|
-
location: string | null;
|
|
872
|
-
url: string | null;
|
|
873
|
-
verified_type: string | null;
|
|
874
|
-
entities_json: string;
|
|
875
|
-
raw_json: string;
|
|
876
|
-
created_at: string;
|
|
877
|
-
};
|
|
878
|
-
const existingProfiles = new Map(
|
|
879
|
-
(
|
|
880
|
-
getNativeDb()
|
|
881
|
-
.prepare(
|
|
882
|
-
`
|
|
923
|
+
verified_type: string | null;
|
|
924
|
+
entities_json: string;
|
|
925
|
+
raw_json: string;
|
|
926
|
+
created_at: string;
|
|
927
|
+
};
|
|
928
|
+
const existingProfiles = new Map(
|
|
929
|
+
(
|
|
930
|
+
getNativeDb()
|
|
931
|
+
.prepare(
|
|
932
|
+
`
|
|
883
933
|
select id, handle, display_name, bio, followers_count, following_count,
|
|
884
934
|
public_metrics_json, avatar_hue, avatar_url, location, url,
|
|
885
935
|
verified_type, entities_json, raw_json, created_at
|
|
886
936
|
from profiles
|
|
887
937
|
`,
|
|
888
|
-
|
|
889
|
-
|
|
890
|
-
|
|
891
|
-
);
|
|
892
|
-
const existingProfilesByHandle = new Map(
|
|
893
|
-
[...existingProfiles.values()].map((profile) => [
|
|
894
|
-
profile.handle.toLowerCase(),
|
|
895
|
-
profile,
|
|
896
|
-
]),
|
|
897
|
-
);
|
|
898
|
-
const existingPrimaryAccount = getNativeDb()
|
|
899
|
-
.prepare("select handle, external_user_id from accounts where id = ?")
|
|
900
|
-
.get("acct_primary") as
|
|
901
|
-
| { handle: string; external_user_id: string | null }
|
|
902
|
-
| undefined;
|
|
903
|
-
const profileIdAliases = new Map<string, string>();
|
|
904
|
-
|
|
905
|
-
type ArchiveProfileTier =
|
|
906
|
-
| "archive_follow_stub"
|
|
907
|
-
| "archive_dm_stub"
|
|
908
|
-
| "archive_mention_inferred"
|
|
909
|
-
| "live_or_hydrated";
|
|
910
|
-
const archiveProfileTierRank: Record<ArchiveProfileTier, number> = {
|
|
911
|
-
archive_follow_stub: 0,
|
|
912
|
-
archive_dm_stub: 1,
|
|
913
|
-
archive_mention_inferred: 2,
|
|
914
|
-
live_or_hydrated: 3,
|
|
915
|
-
};
|
|
916
|
-
|
|
917
|
-
function classifyExistingProfile(profile: ProfileRow): ArchiveProfileTier {
|
|
918
|
-
const externalUserId = profile.id.startsWith("profile_user_")
|
|
919
|
-
? profile.id.slice("profile_user_".length)
|
|
920
|
-
: "";
|
|
921
|
-
const fallbackHandle = externalUserId ? `id${externalUserId}` : profile.id;
|
|
922
|
-
const hasLiveSignals =
|
|
923
|
-
profile.followersCount > 0 ||
|
|
924
|
-
profile.followingCount > 0 ||
|
|
925
|
-
profile.publicMetricsJson.trim() !== "{}" ||
|
|
926
|
-
profile.avatarUrl !== null ||
|
|
927
|
-
profile.location !== null ||
|
|
928
|
-
profile.url !== null ||
|
|
929
|
-
profile.verifiedType !== null ||
|
|
930
|
-
profile.entitiesJson.trim() !== "{}" ||
|
|
931
|
-
profile.rawJson.trim() !== "{}";
|
|
932
|
-
|
|
933
|
-
if (hasLiveSignals) return "live_or_hydrated";
|
|
934
|
-
if (
|
|
935
|
-
profile.handle === fallbackHandle &&
|
|
936
|
-
profile.displayName === "" &&
|
|
937
|
-
profile.bio === ""
|
|
938
|
-
) {
|
|
939
|
-
return "archive_follow_stub";
|
|
940
|
-
}
|
|
941
|
-
if (profile.bio.startsWith("Imported from archive user ")) {
|
|
942
|
-
return profile.handle === fallbackHandle &&
|
|
943
|
-
profile.displayName === fallbackHandle
|
|
944
|
-
? "archive_dm_stub"
|
|
945
|
-
: "archive_mention_inferred";
|
|
946
|
-
}
|
|
947
|
-
return profile.handle === fallbackHandle && profile.displayName === ""
|
|
948
|
-
? "archive_follow_stub"
|
|
949
|
-
: "archive_mention_inferred";
|
|
950
|
-
}
|
|
951
|
-
|
|
952
|
-
function shouldPreserveProfile(
|
|
953
|
-
existingTier: ArchiveProfileTier,
|
|
954
|
-
incomingTier: ArchiveProfileTier,
|
|
955
|
-
) {
|
|
956
|
-
return (
|
|
957
|
-
archiveProfileTierRank[existingTier] >=
|
|
958
|
-
archiveProfileTierRank[incomingTier]
|
|
938
|
+
)
|
|
939
|
+
.all() as ExistingProfileRow[]
|
|
940
|
+
).map((profile) => [profile.id, profile]),
|
|
959
941
|
);
|
|
960
|
-
|
|
961
|
-
|
|
962
|
-
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
|
|
969
|
-
|
|
970
|
-
|
|
971
|
-
|
|
972
|
-
|
|
973
|
-
|
|
974
|
-
|
|
975
|
-
|
|
976
|
-
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
942
|
+
const existingProfilesByHandle = new Map(
|
|
943
|
+
[...existingProfiles.values()].map((profile) => [
|
|
944
|
+
profile.handle.toLowerCase(),
|
|
945
|
+
profile,
|
|
946
|
+
]),
|
|
947
|
+
);
|
|
948
|
+
const existingPrimaryAccount = getNativeDb()
|
|
949
|
+
.prepare("select handle, external_user_id from accounts where id = ?")
|
|
950
|
+
.get("acct_primary") as
|
|
951
|
+
| { handle: string; external_user_id: string | null }
|
|
952
|
+
| undefined;
|
|
953
|
+
const profileIdAliases = new Map<string, string>();
|
|
954
|
+
|
|
955
|
+
type ArchiveProfileTier =
|
|
956
|
+
| "archive_follow_stub"
|
|
957
|
+
| "archive_dm_stub"
|
|
958
|
+
| "archive_mention_inferred"
|
|
959
|
+
| "live_or_hydrated";
|
|
960
|
+
const archiveProfileTierRank: Record<ArchiveProfileTier, number> = {
|
|
961
|
+
archive_follow_stub: 0,
|
|
962
|
+
archive_dm_stub: 1,
|
|
963
|
+
archive_mention_inferred: 2,
|
|
964
|
+
live_or_hydrated: 3,
|
|
981
965
|
};
|
|
982
|
-
}
|
|
983
966
|
|
|
984
|
-
|
|
985
|
-
|
|
986
|
-
|
|
987
|
-
|
|
988
|
-
|
|
989
|
-
|
|
990
|
-
|
|
991
|
-
|
|
992
|
-
|
|
967
|
+
function classifyExistingProfile(profile: ProfileRow): ArchiveProfileTier {
|
|
968
|
+
const externalUserId = profile.id.startsWith("profile_user_")
|
|
969
|
+
? profile.id.slice("profile_user_".length)
|
|
970
|
+
: "";
|
|
971
|
+
const fallbackHandle = externalUserId
|
|
972
|
+
? `id${externalUserId}`
|
|
973
|
+
: profile.id;
|
|
974
|
+
const hasLiveSignals =
|
|
975
|
+
profile.followersCount > 0 ||
|
|
976
|
+
profile.followingCount > 0 ||
|
|
977
|
+
profile.publicMetricsJson.trim() !== "{}" ||
|
|
978
|
+
profile.avatarUrl !== null ||
|
|
979
|
+
profile.location !== null ||
|
|
980
|
+
profile.url !== null ||
|
|
981
|
+
profile.verifiedType !== null ||
|
|
982
|
+
profile.entitiesJson.trim() !== "{}" ||
|
|
983
|
+
profile.rawJson.trim() !== "{}";
|
|
984
|
+
|
|
985
|
+
if (hasLiveSignals) return "live_or_hydrated";
|
|
986
|
+
if (
|
|
987
|
+
profile.handle === fallbackHandle &&
|
|
988
|
+
profile.displayName === "" &&
|
|
989
|
+
profile.bio === ""
|
|
990
|
+
) {
|
|
991
|
+
return "archive_follow_stub";
|
|
992
|
+
}
|
|
993
|
+
if (profile.bio.startsWith("Imported from archive user ")) {
|
|
994
|
+
return profile.handle === fallbackHandle &&
|
|
995
|
+
profile.displayName === fallbackHandle
|
|
996
|
+
? "archive_dm_stub"
|
|
997
|
+
: "archive_mention_inferred";
|
|
998
|
+
}
|
|
999
|
+
return profile.handle === fallbackHandle && profile.displayName === ""
|
|
1000
|
+
? "archive_follow_stub"
|
|
1001
|
+
: "archive_mention_inferred";
|
|
993
1002
|
}
|
|
994
|
-
const targetIncoming =
|
|
995
|
-
targetId === incoming.id ? incoming : { ...incoming, id: targetId };
|
|
996
|
-
const incomingTier = classifyExistingProfile(incoming);
|
|
997
|
-
const current = profiles.get(targetId);
|
|
998
|
-
const currentTier = current ? classifyExistingProfile(current) : null;
|
|
999
|
-
const existingProfile = targetExisting
|
|
1000
|
-
? existingProfileToProfileRow(targetExisting)
|
|
1001
|
-
: null;
|
|
1002
|
-
const existingTier = existingProfile
|
|
1003
|
-
? classifyExistingProfile(existingProfile)
|
|
1004
|
-
: null;
|
|
1005
1003
|
|
|
1006
|
-
|
|
1007
|
-
|
|
1008
|
-
|
|
1009
|
-
shouldPreserveProfile(currentTier, incomingTier) &&
|
|
1010
|
-
(!existingTier || shouldPreserveProfile(currentTier, existingTier))
|
|
1004
|
+
function shouldPreserveProfile(
|
|
1005
|
+
existingTier: ArchiveProfileTier,
|
|
1006
|
+
incomingTier: ArchiveProfileTier,
|
|
1011
1007
|
) {
|
|
1012
|
-
return
|
|
1008
|
+
return (
|
|
1009
|
+
archiveProfileTierRank[existingTier] >=
|
|
1010
|
+
archiveProfileTierRank[incomingTier]
|
|
1011
|
+
);
|
|
1013
1012
|
}
|
|
1014
1013
|
|
|
1015
|
-
|
|
1016
|
-
|
|
1017
|
-
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
|
|
1021
|
-
|
|
1014
|
+
function existingProfileToProfileRow(
|
|
1015
|
+
profile: ExistingProfileRow,
|
|
1016
|
+
): ProfileRow {
|
|
1017
|
+
return {
|
|
1018
|
+
id: profile.id,
|
|
1019
|
+
handle: profile.handle,
|
|
1020
|
+
displayName: profile.display_name,
|
|
1021
|
+
bio: profile.bio,
|
|
1022
|
+
followersCount: profile.followers_count,
|
|
1023
|
+
followingCount: profile.following_count,
|
|
1024
|
+
publicMetricsJson: profile.public_metrics_json,
|
|
1025
|
+
avatarHue: profile.avatar_hue,
|
|
1026
|
+
avatarUrl: profile.avatar_url,
|
|
1027
|
+
location: profile.location,
|
|
1028
|
+
url: profile.url,
|
|
1029
|
+
verifiedType: profile.verified_type,
|
|
1030
|
+
entitiesJson: profile.entities_json,
|
|
1031
|
+
rawJson: profile.raw_json,
|
|
1032
|
+
createdAt: profile.created_at,
|
|
1033
|
+
};
|
|
1022
1034
|
}
|
|
1023
1035
|
|
|
1024
|
-
|
|
1025
|
-
|
|
1036
|
+
function mergeArchiveProfile(incoming: ProfileRow) {
|
|
1037
|
+
const existingById = existingProfiles.get(incoming.id);
|
|
1038
|
+
const existingByHandle = selection
|
|
1039
|
+
? existingProfilesByHandle.get(incoming.handle.toLowerCase())
|
|
1040
|
+
: undefined;
|
|
1041
|
+
const targetExisting = existingById ?? existingByHandle;
|
|
1042
|
+
const targetId = targetExisting?.id ?? incoming.id;
|
|
1043
|
+
if (targetId !== incoming.id) {
|
|
1044
|
+
profileIdAliases.set(incoming.id, targetId);
|
|
1045
|
+
}
|
|
1046
|
+
const targetIncoming =
|
|
1047
|
+
targetId === incoming.id ? incoming : { ...incoming, id: targetId };
|
|
1048
|
+
const incomingTier = classifyExistingProfile(incoming);
|
|
1049
|
+
const current = profiles.get(targetId);
|
|
1050
|
+
const currentTier = current ? classifyExistingProfile(current) : null;
|
|
1051
|
+
const existingProfile = targetExisting
|
|
1052
|
+
? existingProfileToProfileRow(targetExisting)
|
|
1053
|
+
: null;
|
|
1054
|
+
const existingTier = existingProfile
|
|
1055
|
+
? classifyExistingProfile(existingProfile)
|
|
1056
|
+
: null;
|
|
1026
1057
|
|
|
1027
|
-
|
|
1028
|
-
|
|
1029
|
-
|
|
1058
|
+
if (
|
|
1059
|
+
current &&
|
|
1060
|
+
currentTier &&
|
|
1061
|
+
shouldPreserveProfile(currentTier, incomingTier) &&
|
|
1062
|
+
(!existingTier || shouldPreserveProfile(currentTier, existingTier))
|
|
1063
|
+
) {
|
|
1064
|
+
return;
|
|
1065
|
+
}
|
|
1030
1066
|
|
|
1031
|
-
function isProfileHandleTakenByOtherId(handle: string, profileId: string) {
|
|
1032
|
-
const normalizedHandle = handle.toLowerCase();
|
|
1033
|
-
const existingProfile = existingProfilesByHandle.get(normalizedHandle);
|
|
1034
|
-
if (existingProfile && existingProfile.id !== profileId) return true;
|
|
1035
|
-
for (const profile of profiles.values()) {
|
|
1036
1067
|
if (
|
|
1037
|
-
|
|
1038
|
-
|
|
1068
|
+
existingProfile &&
|
|
1069
|
+
existingTier &&
|
|
1070
|
+
shouldPreserveProfile(existingTier, incomingTier)
|
|
1039
1071
|
) {
|
|
1040
|
-
|
|
1072
|
+
profiles.set(targetId, existingProfile);
|
|
1073
|
+
return;
|
|
1041
1074
|
}
|
|
1075
|
+
|
|
1076
|
+
profiles.set(targetId, targetIncoming);
|
|
1042
1077
|
}
|
|
1043
|
-
return false;
|
|
1044
|
-
}
|
|
1045
1078
|
|
|
1046
|
-
|
|
1047
|
-
|
|
1048
|
-
return baseHandle;
|
|
1079
|
+
function resolveProfileId(profileId: string) {
|
|
1080
|
+
return profileIdAliases.get(profileId) ?? profileId;
|
|
1049
1081
|
}
|
|
1050
|
-
|
|
1051
|
-
|
|
1052
|
-
const
|
|
1053
|
-
const
|
|
1054
|
-
if (
|
|
1055
|
-
|
|
1082
|
+
|
|
1083
|
+
function isProfileHandleTakenByOtherId(handle: string, profileId: string) {
|
|
1084
|
+
const normalizedHandle = handle.toLowerCase();
|
|
1085
|
+
const existingProfile = existingProfilesByHandle.get(normalizedHandle);
|
|
1086
|
+
if (existingProfile && existingProfile.id !== profileId) return true;
|
|
1087
|
+
for (const profile of profiles.values()) {
|
|
1088
|
+
if (
|
|
1089
|
+
profile.id !== profileId &&
|
|
1090
|
+
profile.handle.toLowerCase() === normalizedHandle
|
|
1091
|
+
) {
|
|
1092
|
+
return true;
|
|
1093
|
+
}
|
|
1056
1094
|
}
|
|
1057
|
-
|
|
1095
|
+
return false;
|
|
1058
1096
|
}
|
|
1059
|
-
}
|
|
1060
|
-
|
|
1061
|
-
function addArchiveFollowProfile(profileId: string, externalUserId: string) {
|
|
1062
|
-
if (!profileId) return;
|
|
1063
|
-
const fallbackId =
|
|
1064
|
-
externalUserId || profileId.replace(/^profile_user_/, "");
|
|
1065
|
-
mergeArchiveProfile({
|
|
1066
|
-
id: profileId,
|
|
1067
|
-
handle: fallbackId ? `id${fallbackId}` : profileId,
|
|
1068
|
-
displayName: "",
|
|
1069
|
-
bio: "",
|
|
1070
|
-
followersCount: 0,
|
|
1071
|
-
followingCount: 0,
|
|
1072
|
-
...defaultProfileMetadata,
|
|
1073
|
-
avatarHue: 210,
|
|
1074
|
-
avatarUrl: null,
|
|
1075
|
-
createdAt: accountPayload.createdAt,
|
|
1076
|
-
});
|
|
1077
|
-
}
|
|
1078
1097
|
|
|
1079
|
-
|
|
1080
|
-
|
|
1081
|
-
|
|
1082
|
-
|
|
1083
|
-
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1087
|
-
|
|
1088
|
-
|
|
1089
|
-
|
|
1090
|
-
|
|
1091
|
-
|
|
1092
|
-
.toLowerCase();
|
|
1093
|
-
if (
|
|
1094
|
-
!existingExternalUserId &&
|
|
1095
|
-
existingHandle !== accountPayload.username.toLowerCase()
|
|
1096
|
-
) {
|
|
1097
|
-
throw new Error(
|
|
1098
|
-
`Existing acct_primary (@${existingHandle}) does not match archive account @${accountPayload.username}`,
|
|
1099
|
-
);
|
|
1098
|
+
function uniqueArchiveProfileHandle(baseHandle: string, profileId: string) {
|
|
1099
|
+
if (!isProfileHandleTakenByOtherId(baseHandle, profileId)) {
|
|
1100
|
+
return baseHandle;
|
|
1101
|
+
}
|
|
1102
|
+
let index = 1;
|
|
1103
|
+
while (true) {
|
|
1104
|
+
const suffix = index === 1 ? "archive" : `archive_${index}`;
|
|
1105
|
+
const candidate = `${baseHandle}_${suffix}`;
|
|
1106
|
+
if (!isProfileHandleTakenByOtherId(candidate, profileId)) {
|
|
1107
|
+
return candidate;
|
|
1108
|
+
}
|
|
1109
|
+
index += 1;
|
|
1110
|
+
}
|
|
1100
1111
|
}
|
|
1101
|
-
}
|
|
1102
1112
|
|
|
1103
|
-
|
|
1104
|
-
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
|
|
1108
|
-
|
|
1109
|
-
(
|
|
1110
|
-
|
|
1111
|
-
|
|
1112
|
-
|
|
1113
|
-
|
|
1114
|
-
|
|
1115
|
-
...existingProfileToProfileRow(existingLocalProfile),
|
|
1116
|
-
handle: accountPayload.username,
|
|
1117
|
-
displayName: accountPayload.displayName,
|
|
1118
|
-
bio: accountPayload.bio,
|
|
1119
|
-
createdAt: accountPayload.createdAt,
|
|
1120
|
-
}
|
|
1121
|
-
: {
|
|
1122
|
-
id: "profile_me",
|
|
1123
|
-
handle: accountPayload.username,
|
|
1124
|
-
displayName: accountPayload.displayName,
|
|
1125
|
-
bio: accountPayload.bio,
|
|
1113
|
+
function addArchiveFollowProfile(
|
|
1114
|
+
profileId: string,
|
|
1115
|
+
externalUserId: string,
|
|
1116
|
+
) {
|
|
1117
|
+
if (!profileId) return;
|
|
1118
|
+
const fallbackId =
|
|
1119
|
+
externalUserId || profileId.replace(/^profile_user_/, "");
|
|
1120
|
+
mergeArchiveProfile({
|
|
1121
|
+
id: profileId,
|
|
1122
|
+
handle: fallbackId ? `id${fallbackId}` : profileId,
|
|
1123
|
+
displayName: "",
|
|
1124
|
+
bio: "",
|
|
1126
1125
|
followersCount: 0,
|
|
1127
1126
|
followingCount: 0,
|
|
1128
1127
|
...defaultProfileMetadata,
|
|
1129
|
-
avatarHue:
|
|
1128
|
+
avatarHue: 210,
|
|
1130
1129
|
avatarUrl: null,
|
|
1131
1130
|
createdAt: accountPayload.createdAt,
|
|
1132
|
-
};
|
|
1133
|
-
|
|
1134
|
-
|
|
1135
|
-
|
|
1136
|
-
|
|
1137
|
-
|
|
1138
|
-
|
|
1139
|
-
|
|
1140
|
-
|
|
1141
|
-
|
|
1142
|
-
|
|
1143
|
-
|
|
1144
|
-
|
|
1145
|
-
|
|
1146
|
-
|
|
1147
|
-
|
|
1148
|
-
|
|
1149
|
-
|
|
1150
|
-
|
|
1131
|
+
});
|
|
1132
|
+
}
|
|
1133
|
+
|
|
1134
|
+
function assertSelectedAccountMatchesArchive() {
|
|
1135
|
+
if (!selection || !existingPrimaryAccount) return;
|
|
1136
|
+
const existingExternalUserId = existingPrimaryAccount.external_user_id;
|
|
1137
|
+
if (
|
|
1138
|
+
existingExternalUserId &&
|
|
1139
|
+
existingExternalUserId !== accountPayload.accountId
|
|
1140
|
+
) {
|
|
1141
|
+
throw new Error(
|
|
1142
|
+
`Existing acct_primary (${existingExternalUserId}) does not match archive account ${accountPayload.accountId}`,
|
|
1143
|
+
);
|
|
1144
|
+
}
|
|
1145
|
+
const existingHandle = existingPrimaryAccount.handle
|
|
1146
|
+
.replace(/^@/, "")
|
|
1147
|
+
.toLowerCase();
|
|
1148
|
+
if (
|
|
1149
|
+
!existingExternalUserId &&
|
|
1150
|
+
existingHandle !== accountPayload.username.toLowerCase()
|
|
1151
|
+
) {
|
|
1152
|
+
throw new Error(
|
|
1153
|
+
`Existing acct_primary (@${existingHandle}) does not match archive account @${accountPayload.username}`,
|
|
1154
|
+
);
|
|
1155
|
+
}
|
|
1156
|
+
}
|
|
1157
|
+
|
|
1158
|
+
assertSelectedAccountMatchesArchive();
|
|
1159
|
+
|
|
1160
|
+
const existingLocalProfile =
|
|
1161
|
+
selection &&
|
|
1162
|
+
(existingProfiles.get("profile_me") ??
|
|
1163
|
+
[...existingProfiles.values()].find(
|
|
1164
|
+
(profile) =>
|
|
1165
|
+
profile.handle.toLowerCase() ===
|
|
1166
|
+
accountPayload.username.toLowerCase(),
|
|
1167
|
+
));
|
|
1168
|
+
const archivedLocalProfile = existingLocalProfile
|
|
1169
|
+
? {
|
|
1170
|
+
...existingProfileToProfileRow(existingLocalProfile),
|
|
1171
|
+
handle: accountPayload.username,
|
|
1172
|
+
displayName: accountPayload.displayName,
|
|
1173
|
+
bio: accountPayload.bio,
|
|
1174
|
+
createdAt: accountPayload.createdAt,
|
|
1175
|
+
}
|
|
1176
|
+
: {
|
|
1177
|
+
id: "profile_me",
|
|
1178
|
+
handle: accountPayload.username,
|
|
1179
|
+
displayName: accountPayload.displayName,
|
|
1180
|
+
bio: accountPayload.bio,
|
|
1181
|
+
followersCount: 0,
|
|
1182
|
+
followingCount: 0,
|
|
1183
|
+
...defaultProfileMetadata,
|
|
1184
|
+
avatarHue: 18,
|
|
1185
|
+
avatarUrl: null,
|
|
1186
|
+
createdAt: accountPayload.createdAt,
|
|
1187
|
+
};
|
|
1188
|
+
const localProfile =
|
|
1189
|
+
existingLocalProfile && !includeProfiles
|
|
1190
|
+
? existingProfileToProfileRow(existingLocalProfile)
|
|
1191
|
+
: archivedLocalProfile;
|
|
1192
|
+
profiles.set(localProfile.id, localProfile);
|
|
1193
|
+
|
|
1194
|
+
const existingDmConversationAccounts = new Map(
|
|
1195
|
+
(
|
|
1196
|
+
getNativeDb()
|
|
1197
|
+
.prepare("select id, account_id from dm_conversations")
|
|
1198
|
+
.all() as Array<{ id: string; account_id: string }>
|
|
1199
|
+
).map((row) => [row.id, row.account_id]),
|
|
1200
|
+
);
|
|
1201
|
+
const existingOtherDmMessageIds = new Set(
|
|
1202
|
+
(
|
|
1203
|
+
getNativeDb()
|
|
1204
|
+
.prepare(
|
|
1205
|
+
`
|
|
1151
1206
|
select m.id
|
|
1152
1207
|
from dm_messages m
|
|
1153
1208
|
join dm_conversations c on c.id = m.conversation_id
|
|
1154
1209
|
where c.account_id <> 'acct_primary'
|
|
1155
1210
|
`,
|
|
1156
|
-
|
|
1157
|
-
|
|
1158
|
-
|
|
1159
|
-
|
|
1160
|
-
|
|
1161
|
-
|
|
1162
|
-
|
|
1163
|
-
|
|
1164
|
-
|
|
1165
|
-
|
|
1166
|
-
|
|
1167
|
-
|
|
1168
|
-
|
|
1169
|
-
|
|
1170
|
-
|
|
1171
|
-
|
|
1172
|
-
|
|
1173
|
-
|
|
1211
|
+
)
|
|
1212
|
+
.all() as Array<{ id: string }>
|
|
1213
|
+
).map((row) => row.id),
|
|
1214
|
+
);
|
|
1215
|
+
const archiveDmConversationIdAliases = new Map<string, string>();
|
|
1216
|
+
const archiveDmMessageIdAliases = new Map<string, string>();
|
|
1217
|
+
|
|
1218
|
+
function uniquePrimaryArchiveId(
|
|
1219
|
+
baseId: string,
|
|
1220
|
+
isTakenByOtherAccount: (candidate: string) => boolean,
|
|
1221
|
+
isPending: (candidate: string) => boolean,
|
|
1222
|
+
) {
|
|
1223
|
+
let index = 1;
|
|
1224
|
+
while (true) {
|
|
1225
|
+
const suffix = index === 1 ? "" : `:${index}`;
|
|
1226
|
+
const candidate = `acct_primary:${baseId}${suffix}`;
|
|
1227
|
+
if (!isTakenByOtherAccount(candidate) && !isPending(candidate)) {
|
|
1228
|
+
return candidate;
|
|
1229
|
+
}
|
|
1230
|
+
index += 1;
|
|
1174
1231
|
}
|
|
1175
|
-
index += 1;
|
|
1176
1232
|
}
|
|
1177
|
-
}
|
|
1178
1233
|
|
|
1179
|
-
|
|
1180
|
-
|
|
1181
|
-
|
|
1182
|
-
|
|
1183
|
-
|
|
1184
|
-
|
|
1185
|
-
|
|
1234
|
+
function resolveArchiveDmConversationId(conversationId: string) {
|
|
1235
|
+
const existingAlias = archiveDmConversationIdAliases.get(conversationId);
|
|
1236
|
+
if (existingAlias) return existingAlias;
|
|
1237
|
+
if (!selection) {
|
|
1238
|
+
archiveDmConversationIdAliases.set(conversationId, conversationId);
|
|
1239
|
+
return conversationId;
|
|
1240
|
+
}
|
|
1186
1241
|
|
|
1187
|
-
|
|
1188
|
-
|
|
1189
|
-
|
|
1190
|
-
|
|
1191
|
-
|
|
1192
|
-
|
|
1193
|
-
|
|
1194
|
-
|
|
1195
|
-
|
|
1196
|
-
|
|
1197
|
-
|
|
1198
|
-
|
|
1199
|
-
|
|
1200
|
-
|
|
1242
|
+
const takenByOtherAccount = (candidate: string) => {
|
|
1243
|
+
const accountId = existingDmConversationAccounts.get(candidate);
|
|
1244
|
+
return accountId !== undefined && accountId !== "acct_primary";
|
|
1245
|
+
};
|
|
1246
|
+
const resolved = takenByOtherAccount(conversationId)
|
|
1247
|
+
? uniquePrimaryArchiveId(
|
|
1248
|
+
conversationId,
|
|
1249
|
+
takenByOtherAccount,
|
|
1250
|
+
(candidate) => conversations.has(candidate),
|
|
1251
|
+
)
|
|
1252
|
+
: conversationId;
|
|
1253
|
+
archiveDmConversationIdAliases.set(conversationId, resolved);
|
|
1254
|
+
return resolved;
|
|
1255
|
+
}
|
|
1201
1256
|
|
|
1202
|
-
|
|
1203
|
-
|
|
1204
|
-
|
|
1205
|
-
|
|
1206
|
-
|
|
1207
|
-
|
|
1208
|
-
|
|
1209
|
-
|
|
1210
|
-
|
|
1211
|
-
|
|
1212
|
-
|
|
1213
|
-
|
|
1214
|
-
|
|
1215
|
-
|
|
1216
|
-
|
|
1217
|
-
|
|
1218
|
-
|
|
1219
|
-
|
|
1220
|
-
|
|
1257
|
+
function resolveArchiveDmMessageId(
|
|
1258
|
+
messageId: string,
|
|
1259
|
+
conversationIdChanged: boolean,
|
|
1260
|
+
) {
|
|
1261
|
+
const existingAlias = archiveDmMessageIdAliases.get(messageId);
|
|
1262
|
+
if (existingAlias) return existingAlias;
|
|
1263
|
+
const shouldRemap =
|
|
1264
|
+
selection &&
|
|
1265
|
+
(conversationIdChanged || existingOtherDmMessageIds.has(messageId));
|
|
1266
|
+
const resolved = shouldRemap
|
|
1267
|
+
? uniquePrimaryArchiveId(
|
|
1268
|
+
messageId,
|
|
1269
|
+
(candidate) => existingOtherDmMessageIds.has(candidate),
|
|
1270
|
+
(candidate) =>
|
|
1271
|
+
dmMessages.some((message) => message.id === candidate),
|
|
1272
|
+
)
|
|
1273
|
+
: messageId;
|
|
1274
|
+
archiveDmMessageIdAliases.set(messageId, resolved);
|
|
1275
|
+
return resolved;
|
|
1276
|
+
}
|
|
1221
1277
|
|
|
1222
|
-
|
|
1223
|
-
|
|
1224
|
-
|
|
1225
|
-
|
|
1226
|
-
|
|
1227
|
-
|
|
1228
|
-
|
|
1229
|
-
|
|
1230
|
-
|
|
1231
|
-
|
|
1232
|
-
|
|
1233
|
-
|
|
1234
|
-
|
|
1235
|
-
|
|
1236
|
-
|
|
1237
|
-
|
|
1278
|
+
for (const entry of dmEntries) {
|
|
1279
|
+
const content = yield* readArchiveEntryEffect(archivePath, entry);
|
|
1280
|
+
for (const wrapper of parseArchiveArray(content)) {
|
|
1281
|
+
const dmConversation = asRecord(wrapper.dmConversation);
|
|
1282
|
+
if (!dmConversation) continue;
|
|
1283
|
+
|
|
1284
|
+
const rawConversationId = String(dmConversation.conversationId ?? "");
|
|
1285
|
+
if (!rawConversationId) continue;
|
|
1286
|
+
const conversationId =
|
|
1287
|
+
resolveArchiveDmConversationId(rawConversationId);
|
|
1288
|
+
const conversationIdChanged = conversationId !== rawConversationId;
|
|
1289
|
+
|
|
1290
|
+
const conversationName = String(dmConversation.name ?? "").trim();
|
|
1291
|
+
const participantIds = new Set<string>();
|
|
1292
|
+
const rawMessages = asArray<Record<string, unknown>>(
|
|
1293
|
+
dmConversation.messages,
|
|
1294
|
+
);
|
|
1238
1295
|
|
|
1239
|
-
|
|
1240
|
-
|
|
1241
|
-
|
|
1242
|
-
|
|
1243
|
-
|
|
1244
|
-
|
|
1245
|
-
|
|
1246
|
-
|
|
1296
|
+
for (const event of rawMessages) {
|
|
1297
|
+
const messageCreate = asRecord(event.messageCreate);
|
|
1298
|
+
if (messageCreate) {
|
|
1299
|
+
const senderId = String(messageCreate.senderId ?? "");
|
|
1300
|
+
const recipientId = String(messageCreate.recipientId ?? "");
|
|
1301
|
+
if (senderId) participantIds.add(senderId);
|
|
1302
|
+
if (recipientId) participantIds.add(recipientId);
|
|
1303
|
+
}
|
|
1247
1304
|
|
|
1248
|
-
|
|
1249
|
-
|
|
1250
|
-
|
|
1251
|
-
|
|
1252
|
-
|
|
1253
|
-
|
|
1305
|
+
const joinConversation = asRecord(event.joinConversation);
|
|
1306
|
+
if (joinConversation) {
|
|
1307
|
+
for (const userId of asArray<string>(
|
|
1308
|
+
joinConversation.participantsSnapshot,
|
|
1309
|
+
)) {
|
|
1310
|
+
participantIds.add(String(userId));
|
|
1311
|
+
}
|
|
1254
1312
|
}
|
|
1255
|
-
}
|
|
1256
1313
|
|
|
1257
|
-
|
|
1258
|
-
|
|
1259
|
-
|
|
1260
|
-
|
|
1314
|
+
const participantsJoin = asRecord(event.participantsJoin);
|
|
1315
|
+
if (participantsJoin) {
|
|
1316
|
+
for (const userId of asArray<string>(participantsJoin.userIds)) {
|
|
1317
|
+
participantIds.add(String(userId));
|
|
1318
|
+
}
|
|
1319
|
+
const initiatingUserId = String(
|
|
1320
|
+
participantsJoin.initiatingUserId ?? "",
|
|
1321
|
+
);
|
|
1322
|
+
if (initiatingUserId) {
|
|
1323
|
+
participantIds.add(initiatingUserId);
|
|
1324
|
+
}
|
|
1261
1325
|
}
|
|
1262
|
-
|
|
1263
|
-
|
|
1264
|
-
)
|
|
1265
|
-
|
|
1266
|
-
|
|
1326
|
+
|
|
1327
|
+
const participantsLeave = asRecord(event.participantsLeave);
|
|
1328
|
+
if (participantsLeave) {
|
|
1329
|
+
for (const userId of asArray<string>(participantsLeave.userIds)) {
|
|
1330
|
+
participantIds.add(String(userId));
|
|
1331
|
+
}
|
|
1332
|
+
const initiatingUserId = String(
|
|
1333
|
+
participantsLeave.initiatingUserId ?? "",
|
|
1334
|
+
);
|
|
1335
|
+
if (initiatingUserId) {
|
|
1336
|
+
participantIds.add(initiatingUserId);
|
|
1337
|
+
}
|
|
1267
1338
|
}
|
|
1268
1339
|
}
|
|
1269
1340
|
|
|
1270
|
-
const
|
|
1271
|
-
|
|
1272
|
-
|
|
1273
|
-
|
|
1274
|
-
|
|
1275
|
-
|
|
1276
|
-
|
|
1277
|
-
|
|
1278
|
-
|
|
1279
|
-
|
|
1341
|
+
const externalParticipantIds = [...participantIds].filter(
|
|
1342
|
+
(userId) => userId && userId !== accountPayload.accountId,
|
|
1343
|
+
);
|
|
1344
|
+
const isGroup =
|
|
1345
|
+
conversationName.length > 0 || externalParticipantIds.length > 1;
|
|
1346
|
+
const participantProfileId = isGroup
|
|
1347
|
+
? `profile_group_${conversationId}`
|
|
1348
|
+
: `profile_user_${externalParticipantIds[0] ?? conversationId}`;
|
|
1349
|
+
|
|
1350
|
+
if (!profiles.has(participantProfileId)) {
|
|
1351
|
+
if (isGroup) {
|
|
1352
|
+
profiles.set(participantProfileId, {
|
|
1353
|
+
id: participantProfileId,
|
|
1354
|
+
handle: `group-${conversationId}`,
|
|
1355
|
+
displayName:
|
|
1356
|
+
conversationName || `Group DM ${externalParticipantIds.length}`,
|
|
1357
|
+
bio: `Group DM with ${externalParticipantIds.length} participants`,
|
|
1358
|
+
followersCount: 0,
|
|
1359
|
+
followingCount: 0,
|
|
1360
|
+
...defaultProfileMetadata,
|
|
1361
|
+
avatarHue: 220,
|
|
1362
|
+
avatarUrl: null,
|
|
1363
|
+
createdAt: accountPayload.createdAt,
|
|
1364
|
+
});
|
|
1365
|
+
} else {
|
|
1366
|
+
const otherUserId = externalParticipantIds[0] ?? conversationId;
|
|
1367
|
+
const inferred = inferProfileFromDirectory(
|
|
1368
|
+
otherUserId,
|
|
1369
|
+
mentionDirectory,
|
|
1370
|
+
);
|
|
1371
|
+
mergeArchiveProfile({
|
|
1372
|
+
id: participantProfileId,
|
|
1373
|
+
handle: inferred.handle,
|
|
1374
|
+
displayName: inferred.displayName,
|
|
1375
|
+
bio: `Imported from archive user ${otherUserId}`,
|
|
1376
|
+
followersCount: 0,
|
|
1377
|
+
followingCount: 0,
|
|
1378
|
+
...defaultProfileMetadata,
|
|
1379
|
+
avatarHue: 210,
|
|
1380
|
+
avatarUrl: null,
|
|
1381
|
+
createdAt: accountPayload.createdAt,
|
|
1382
|
+
});
|
|
1280
1383
|
}
|
|
1281
1384
|
}
|
|
1282
|
-
}
|
|
1283
1385
|
|
|
1284
|
-
|
|
1285
|
-
|
|
1286
|
-
|
|
1287
|
-
|
|
1288
|
-
|
|
1289
|
-
|
|
1290
|
-
|
|
1291
|
-
|
|
1292
|
-
|
|
1293
|
-
|
|
1294
|
-
|
|
1295
|
-
|
|
1296
|
-
|
|
1297
|
-
|
|
1298
|
-
|
|
1299
|
-
|
|
1300
|
-
|
|
1301
|
-
|
|
1302
|
-
|
|
1303
|
-
|
|
1304
|
-
|
|
1305
|
-
|
|
1306
|
-
|
|
1307
|
-
|
|
1308
|
-
|
|
1309
|
-
|
|
1310
|
-
|
|
1311
|
-
|
|
1312
|
-
|
|
1313
|
-
|
|
1314
|
-
|
|
1315
|
-
|
|
1316
|
-
|
|
1317
|
-
displayName: inferred.displayName,
|
|
1318
|
-
bio: `Imported from archive user ${otherUserId}`,
|
|
1319
|
-
followersCount: 0,
|
|
1320
|
-
followingCount: 0,
|
|
1321
|
-
...defaultProfileMetadata,
|
|
1322
|
-
avatarHue: 210,
|
|
1323
|
-
avatarUrl: null,
|
|
1324
|
-
createdAt: accountPayload.createdAt,
|
|
1325
|
-
});
|
|
1326
|
-
}
|
|
1327
|
-
}
|
|
1386
|
+
const messageEvents = rawMessages
|
|
1387
|
+
.map((event) => asRecord(event.messageCreate))
|
|
1388
|
+
.filter((event): event is Record<string, unknown> => event !== null)
|
|
1389
|
+
.map((messageCreate) => {
|
|
1390
|
+
const senderId = String(messageCreate.senderId ?? "");
|
|
1391
|
+
const rawMessageId = String(
|
|
1392
|
+
messageCreate.id ?? `${rawConversationId}-${senderId}`,
|
|
1393
|
+
);
|
|
1394
|
+
const senderProfileId =
|
|
1395
|
+
senderId === accountPayload.accountId
|
|
1396
|
+
? localProfile.id
|
|
1397
|
+
: `profile_user_${senderId}`;
|
|
1398
|
+
|
|
1399
|
+
if (senderId && senderId !== accountPayload.accountId) {
|
|
1400
|
+
const inferred = inferProfileFromDirectory(
|
|
1401
|
+
senderId,
|
|
1402
|
+
mentionDirectory,
|
|
1403
|
+
);
|
|
1404
|
+
if (!profiles.has(senderProfileId)) {
|
|
1405
|
+
mergeArchiveProfile({
|
|
1406
|
+
id: senderProfileId,
|
|
1407
|
+
handle: inferred.handle,
|
|
1408
|
+
displayName: inferred.displayName,
|
|
1409
|
+
bio: `Imported from archive user ${senderId}`,
|
|
1410
|
+
followersCount: 0,
|
|
1411
|
+
followingCount: 0,
|
|
1412
|
+
...defaultProfileMetadata,
|
|
1413
|
+
avatarHue: 240,
|
|
1414
|
+
avatarUrl: null,
|
|
1415
|
+
createdAt: accountPayload.createdAt,
|
|
1416
|
+
});
|
|
1417
|
+
}
|
|
1418
|
+
}
|
|
1328
1419
|
|
|
1329
|
-
|
|
1330
|
-
|
|
1331
|
-
|
|
1332
|
-
|
|
1333
|
-
|
|
1334
|
-
|
|
1335
|
-
|
|
1420
|
+
return {
|
|
1421
|
+
id: resolveArchiveDmMessageId(
|
|
1422
|
+
rawMessageId,
|
|
1423
|
+
conversationIdChanged,
|
|
1424
|
+
),
|
|
1425
|
+
conversationId,
|
|
1426
|
+
senderProfileId: resolveProfileId(senderProfileId),
|
|
1427
|
+
text: String(messageCreate.text ?? ""),
|
|
1428
|
+
createdAt: parseTwitterDate(messageCreate.createdAt),
|
|
1429
|
+
direction:
|
|
1430
|
+
senderId === accountPayload.accountId ? "outbound" : "inbound",
|
|
1431
|
+
mediaCount: asArray(messageCreate.mediaUrls).length,
|
|
1432
|
+
} satisfies MessageRow;
|
|
1433
|
+
})
|
|
1434
|
+
.sort((left, right) =>
|
|
1435
|
+
compareIsoTimestamp(left.createdAt, right.createdAt),
|
|
1336
1436
|
);
|
|
1337
|
-
const senderProfileId =
|
|
1338
|
-
senderId === accountPayload.accountId
|
|
1339
|
-
? localProfile.id
|
|
1340
|
-
: `profile_user_${senderId}`;
|
|
1341
1437
|
|
|
1342
|
-
|
|
1343
|
-
|
|
1344
|
-
|
|
1345
|
-
mentionDirectory,
|
|
1346
|
-
);
|
|
1347
|
-
if (!profiles.has(senderProfileId)) {
|
|
1348
|
-
mergeArchiveProfile({
|
|
1349
|
-
id: senderProfileId,
|
|
1350
|
-
handle: inferred.handle,
|
|
1351
|
-
displayName: inferred.displayName,
|
|
1352
|
-
bio: `Imported from archive user ${senderId}`,
|
|
1353
|
-
followersCount: 0,
|
|
1354
|
-
followingCount: 0,
|
|
1355
|
-
...defaultProfileMetadata,
|
|
1356
|
-
avatarHue: 240,
|
|
1357
|
-
avatarUrl: null,
|
|
1358
|
-
createdAt: accountPayload.createdAt,
|
|
1359
|
-
});
|
|
1360
|
-
}
|
|
1361
|
-
}
|
|
1438
|
+
if (messageEvents.length === 0) {
|
|
1439
|
+
continue;
|
|
1440
|
+
}
|
|
1362
1441
|
|
|
1363
|
-
|
|
1364
|
-
|
|
1442
|
+
const lastMessage = messageEvents.at(-1);
|
|
1443
|
+
if (!lastMessage) continue;
|
|
1444
|
+
|
|
1445
|
+
dmMessages.push(...messageEvents);
|
|
1446
|
+
const resolvedParticipantProfileId =
|
|
1447
|
+
resolveProfileId(participantProfileId);
|
|
1448
|
+
conversations.set(conversationId, {
|
|
1449
|
+
id: conversationId,
|
|
1450
|
+
title:
|
|
1451
|
+
profiles.get(resolvedParticipantProfileId)?.displayName ||
|
|
1452
|
+
conversationName ||
|
|
1365
1453
|
conversationId,
|
|
1366
|
-
|
|
1367
|
-
|
|
1368
|
-
|
|
1369
|
-
|
|
1370
|
-
|
|
1371
|
-
|
|
1372
|
-
} satisfies MessageRow;
|
|
1373
|
-
})
|
|
1374
|
-
.sort((left, right) =>
|
|
1375
|
-
compareIsoTimestamp(left.createdAt, right.createdAt),
|
|
1376
|
-
);
|
|
1377
|
-
|
|
1378
|
-
if (messageEvents.length === 0) {
|
|
1379
|
-
continue;
|
|
1454
|
+
accountId: "acct_primary",
|
|
1455
|
+
participantProfileId: resolvedParticipantProfileId,
|
|
1456
|
+
lastMessageAt: lastMessage.createdAt,
|
|
1457
|
+
unreadCount: 0,
|
|
1458
|
+
needsReply: lastMessage.direction === "inbound" ? 1 : 0,
|
|
1459
|
+
});
|
|
1380
1460
|
}
|
|
1381
|
-
|
|
1382
|
-
const lastMessage = messageEvents.at(-1);
|
|
1383
|
-
if (!lastMessage) continue;
|
|
1384
|
-
|
|
1385
|
-
dmMessages.push(...messageEvents);
|
|
1386
|
-
const resolvedParticipantProfileId =
|
|
1387
|
-
resolveProfileId(participantProfileId);
|
|
1388
|
-
conversations.set(conversationId, {
|
|
1389
|
-
id: conversationId,
|
|
1390
|
-
title:
|
|
1391
|
-
profiles.get(resolvedParticipantProfileId)?.displayName ||
|
|
1392
|
-
conversationName ||
|
|
1393
|
-
conversationId,
|
|
1394
|
-
accountId: "acct_primary",
|
|
1395
|
-
participantProfileId: resolvedParticipantProfileId,
|
|
1396
|
-
lastMessageAt: lastMessage.createdAt,
|
|
1397
|
-
unreadCount: 0,
|
|
1398
|
-
needsReply: lastMessage.direction === "inbound" ? 1 : 0,
|
|
1399
|
-
});
|
|
1400
1461
|
}
|
|
1401
|
-
}
|
|
1402
1462
|
|
|
1403
|
-
|
|
1404
|
-
|
|
1405
|
-
|
|
1406
|
-
|
|
1407
|
-
|
|
1408
|
-
|
|
1409
|
-
|
|
1410
|
-
|
|
1411
|
-
|
|
1412
|
-
|
|
1413
|
-
|
|
1414
|
-
|
|
1415
|
-
|
|
1416
|
-
|
|
1417
|
-
|
|
1418
|
-
|
|
1419
|
-
|
|
1420
|
-
|
|
1421
|
-
|
|
1422
|
-
|
|
1423
|
-
|
|
1424
|
-
|
|
1425
|
-
|
|
1426
|
-
|
|
1427
|
-
|
|
1428
|
-
|
|
1429
|
-
|
|
1430
|
-
|
|
1431
|
-
|
|
1432
|
-
|
|
1463
|
+
let likeCount = 0;
|
|
1464
|
+
for (const entry of likeEntries) {
|
|
1465
|
+
const content = yield* readArchiveEntryEffect(archivePath, entry);
|
|
1466
|
+
const likes = parseArchiveArray(content);
|
|
1467
|
+
for (const like of likes) {
|
|
1468
|
+
const tweet = extractCollectionTweet(like, "like");
|
|
1469
|
+
if (!tweet) continue;
|
|
1470
|
+
collectionRows.push({
|
|
1471
|
+
tweetId: tweet.id,
|
|
1472
|
+
kind: "likes",
|
|
1473
|
+
collectedAt: tweet.createdAt,
|
|
1474
|
+
source: "archive",
|
|
1475
|
+
rawJson: JSON.stringify(like),
|
|
1476
|
+
});
|
|
1477
|
+
addTweetRow({
|
|
1478
|
+
id: tweet.id,
|
|
1479
|
+
kind: "like",
|
|
1480
|
+
authorProfileId: "profile_unknown",
|
|
1481
|
+
text: tweet.text,
|
|
1482
|
+
createdAt: tweet.createdAt,
|
|
1483
|
+
isReplied: 0,
|
|
1484
|
+
replyToId: null,
|
|
1485
|
+
likeCount: tweet.likeCount,
|
|
1486
|
+
mediaCount: 0,
|
|
1487
|
+
bookmarked: 0,
|
|
1488
|
+
liked: 1,
|
|
1489
|
+
entitiesJson: "{}",
|
|
1490
|
+
mediaJson: "[]",
|
|
1491
|
+
quotedTweetId: null,
|
|
1492
|
+
});
|
|
1493
|
+
}
|
|
1494
|
+
likeCount += likes.length;
|
|
1433
1495
|
}
|
|
1434
|
-
likeCount += likes.length;
|
|
1435
|
-
}
|
|
1436
1496
|
|
|
1437
|
-
|
|
1438
|
-
|
|
1439
|
-
|
|
1440
|
-
|
|
1441
|
-
|
|
1442
|
-
|
|
1443
|
-
|
|
1444
|
-
|
|
1445
|
-
|
|
1446
|
-
|
|
1447
|
-
|
|
1448
|
-
|
|
1449
|
-
|
|
1450
|
-
|
|
1451
|
-
|
|
1452
|
-
|
|
1453
|
-
|
|
1454
|
-
|
|
1455
|
-
|
|
1456
|
-
|
|
1457
|
-
|
|
1458
|
-
|
|
1459
|
-
|
|
1460
|
-
|
|
1461
|
-
|
|
1462
|
-
|
|
1463
|
-
|
|
1464
|
-
|
|
1465
|
-
|
|
1466
|
-
|
|
1497
|
+
let bookmarkCount = 0;
|
|
1498
|
+
for (const entry of bookmarkEntries) {
|
|
1499
|
+
const content = yield* readArchiveEntryEffect(archivePath, entry);
|
|
1500
|
+
const bookmarks = parseArchiveArray(content);
|
|
1501
|
+
for (const bookmark of bookmarks) {
|
|
1502
|
+
const tweet = extractCollectionTweet(bookmark, "bookmark");
|
|
1503
|
+
if (!tweet) continue;
|
|
1504
|
+
collectionRows.push({
|
|
1505
|
+
tweetId: tweet.id,
|
|
1506
|
+
kind: "bookmarks",
|
|
1507
|
+
collectedAt: tweet.createdAt,
|
|
1508
|
+
source: "archive",
|
|
1509
|
+
rawJson: JSON.stringify(bookmark),
|
|
1510
|
+
});
|
|
1511
|
+
addTweetRow({
|
|
1512
|
+
id: tweet.id,
|
|
1513
|
+
kind: "bookmark",
|
|
1514
|
+
authorProfileId: "profile_unknown",
|
|
1515
|
+
text: tweet.text,
|
|
1516
|
+
createdAt: tweet.createdAt,
|
|
1517
|
+
isReplied: 0,
|
|
1518
|
+
replyToId: null,
|
|
1519
|
+
likeCount: tweet.likeCount,
|
|
1520
|
+
mediaCount: 0,
|
|
1521
|
+
bookmarked: 1,
|
|
1522
|
+
liked: 0,
|
|
1523
|
+
entitiesJson: "{}",
|
|
1524
|
+
mediaJson: "[]",
|
|
1525
|
+
quotedTweetId: null,
|
|
1526
|
+
});
|
|
1527
|
+
}
|
|
1528
|
+
bookmarkCount += bookmarks.length;
|
|
1467
1529
|
}
|
|
1468
|
-
bookmarkCount += bookmarks.length;
|
|
1469
|
-
}
|
|
1470
1530
|
|
|
1471
|
-
|
|
1472
|
-
|
|
1473
|
-
|
|
1474
|
-
|
|
1531
|
+
const mediaFileCounts = yield* extractArchiveMediaFilesEffect(
|
|
1532
|
+
archivePath,
|
|
1533
|
+
selectedArchiveMediaKinds(selection),
|
|
1534
|
+
);
|
|
1475
1535
|
|
|
1476
|
-
|
|
1477
|
-
|
|
1478
|
-
|
|
1479
|
-
|
|
1480
|
-
|
|
1481
|
-
|
|
1536
|
+
for (const entry of followerEntries) {
|
|
1537
|
+
const content = yield* readArchiveEntryEffect(archivePath, entry);
|
|
1538
|
+
for (const row of getArchiveFollowRows(content, "follower")) {
|
|
1539
|
+
if (followerIds.has(row.externalUserId)) continue;
|
|
1540
|
+
followerIds.add(row.externalUserId);
|
|
1541
|
+
followerRows.push(row);
|
|
1542
|
+
}
|
|
1482
1543
|
}
|
|
1483
|
-
}
|
|
1484
1544
|
|
|
1485
|
-
|
|
1486
|
-
|
|
1487
|
-
|
|
1488
|
-
|
|
1489
|
-
|
|
1490
|
-
|
|
1545
|
+
for (const entry of followingEntries) {
|
|
1546
|
+
const content = yield* readArchiveEntryEffect(archivePath, entry);
|
|
1547
|
+
for (const row of getArchiveFollowRows(content, "following")) {
|
|
1548
|
+
if (followingIds.has(row.externalUserId)) continue;
|
|
1549
|
+
followingIds.add(row.externalUserId);
|
|
1550
|
+
followingRows.push(row);
|
|
1551
|
+
}
|
|
1491
1552
|
}
|
|
1492
|
-
}
|
|
1493
1553
|
|
|
1494
|
-
|
|
1495
|
-
|
|
1496
|
-
|
|
1554
|
+
for (const row of [...followerRows, ...followingRows]) {
|
|
1555
|
+
addArchiveFollowProfile(row.profileId, row.externalUserId);
|
|
1556
|
+
}
|
|
1497
1557
|
|
|
1498
|
-
|
|
1499
|
-
|
|
1500
|
-
|
|
1501
|
-
|
|
1502
|
-
|
|
1503
|
-
|
|
1504
|
-
|
|
1505
|
-
|
|
1506
|
-
|
|
1507
|
-
|
|
1558
|
+
const clearedFollowDirections = new Set<ArchiveFollowDirection>();
|
|
1559
|
+
if (includeFollowers && followerEntries.length === 0) {
|
|
1560
|
+
clearedFollowDirections.add("followers");
|
|
1561
|
+
}
|
|
1562
|
+
if (includeFollowing && followingEntries.length === 0) {
|
|
1563
|
+
clearedFollowDirections.add("following");
|
|
1564
|
+
}
|
|
1565
|
+
const retainedFollowProfiles = getNativeDb()
|
|
1566
|
+
.prepare(
|
|
1567
|
+
`
|
|
1508
1568
|
select direction, profile_id, external_user_id, source, null as snapshot_id, null as snapshot_source
|
|
1509
1569
|
from follow_edges
|
|
1510
1570
|
union
|
|
@@ -1512,57 +1572,54 @@ export async function importArchive(
|
|
|
1512
1572
|
from follow_events ev
|
|
1513
1573
|
left join follow_snapshots snap on snap.id = ev.snapshot_id
|
|
1514
1574
|
`,
|
|
1515
|
-
|
|
1516
|
-
|
|
1517
|
-
|
|
1518
|
-
|
|
1519
|
-
|
|
1520
|
-
|
|
1521
|
-
|
|
1522
|
-
|
|
1523
|
-
|
|
1524
|
-
|
|
1525
|
-
|
|
1526
|
-
|
|
1527
|
-
|
|
1528
|
-
|
|
1529
|
-
|
|
1530
|
-
|
|
1531
|
-
|
|
1532
|
-
|
|
1533
|
-
|
|
1534
|
-
|
|
1535
|
-
if (tweetRows.some((tweet) => tweet.authorProfileId === "profile_unknown")) {
|
|
1536
|
-
const unknownProfile = {
|
|
1537
|
-
id: "profile_unknown",
|
|
1538
|
-
handle: selection
|
|
1539
|
-
? uniqueArchiveProfileHandle("unknown", "profile_unknown")
|
|
1540
|
-
: "unknown",
|
|
1541
|
-
displayName: "Unknown",
|
|
1542
|
-
bio: "Imported from archive collection metadata",
|
|
1543
|
-
followersCount: 0,
|
|
1544
|
-
followingCount: 0,
|
|
1545
|
-
...defaultProfileMetadata,
|
|
1546
|
-
avatarHue: 210,
|
|
1547
|
-
avatarUrl: null,
|
|
1548
|
-
createdAt: accountPayload.createdAt,
|
|
1549
|
-
};
|
|
1550
|
-
const existingUnknownProfile = existingProfiles.get("profile_unknown");
|
|
1551
|
-
profiles.set(
|
|
1552
|
-
"profile_unknown",
|
|
1553
|
-
existingUnknownProfile
|
|
1554
|
-
? existingProfileToProfileRow(existingUnknownProfile)
|
|
1555
|
-
: unknownProfile,
|
|
1556
|
-
);
|
|
1557
|
-
}
|
|
1575
|
+
)
|
|
1576
|
+
.all() as Array<{
|
|
1577
|
+
direction: ArchiveFollowDirection;
|
|
1578
|
+
profile_id: string;
|
|
1579
|
+
external_user_id: string;
|
|
1580
|
+
source: string | null;
|
|
1581
|
+
snapshot_id: string | null;
|
|
1582
|
+
snapshot_source: string | null;
|
|
1583
|
+
}>;
|
|
1584
|
+
for (const row of retainedFollowProfiles) {
|
|
1585
|
+
const isClearedArchiveRow =
|
|
1586
|
+
clearedFollowDirections.has(row.direction) &&
|
|
1587
|
+
(row.source === "archive" ||
|
|
1588
|
+
row.snapshot_source === "archive" ||
|
|
1589
|
+
row.snapshot_id ===
|
|
1590
|
+
`follow_snapshot_archive_acct_primary_${row.direction}`);
|
|
1591
|
+
if (isClearedArchiveRow) continue;
|
|
1592
|
+
addArchiveFollowProfile(row.profile_id, row.external_user_id);
|
|
1593
|
+
}
|
|
1558
1594
|
|
|
1559
|
-
|
|
1560
|
-
|
|
1561
|
-
|
|
1562
|
-
|
|
1595
|
+
if (
|
|
1596
|
+
tweetRows.some((tweet) => tweet.authorProfileId === "profile_unknown")
|
|
1597
|
+
) {
|
|
1598
|
+
const unknownProfile = {
|
|
1599
|
+
id: "profile_unknown",
|
|
1600
|
+
handle: selection
|
|
1601
|
+
? uniqueArchiveProfileHandle("unknown", "profile_unknown")
|
|
1602
|
+
: "unknown",
|
|
1603
|
+
displayName: "Unknown",
|
|
1604
|
+
bio: "Imported from archive collection metadata",
|
|
1605
|
+
followersCount: 0,
|
|
1606
|
+
followingCount: 0,
|
|
1607
|
+
...defaultProfileMetadata,
|
|
1608
|
+
avatarHue: 210,
|
|
1609
|
+
avatarUrl: null,
|
|
1610
|
+
createdAt: accountPayload.createdAt,
|
|
1611
|
+
};
|
|
1612
|
+
const existingUnknownProfile = existingProfiles.get("profile_unknown");
|
|
1613
|
+
profiles.set(
|
|
1614
|
+
"profile_unknown",
|
|
1615
|
+
existingUnknownProfile
|
|
1616
|
+
? existingProfileToProfileRow(existingUnknownProfile)
|
|
1617
|
+
: unknownProfile,
|
|
1618
|
+
);
|
|
1619
|
+
}
|
|
1563
1620
|
|
|
1564
|
-
|
|
1565
|
-
|
|
1621
|
+
const db = getNativeDb();
|
|
1622
|
+
const insertAccount = db.prepare(`
|
|
1566
1623
|
insert into accounts (id, name, handle, external_user_id, transport, is_default, created_at)
|
|
1567
1624
|
values (?, ?, ?, ?, ?, 1, ?)
|
|
1568
1625
|
on conflict(id) do update set
|
|
@@ -1573,11 +1630,11 @@ export async function importArchive(
|
|
|
1573
1630
|
is_default = 1,
|
|
1574
1631
|
created_at = excluded.created_at
|
|
1575
1632
|
`);
|
|
1576
|
-
|
|
1633
|
+
const insertAccountIfMissing = db.prepare(`
|
|
1577
1634
|
insert or ignore into accounts (id, name, handle, external_user_id, transport, is_default, created_at)
|
|
1578
1635
|
values (?, ?, ?, ?, ?, 1, ?)
|
|
1579
1636
|
`);
|
|
1580
|
-
|
|
1637
|
+
const insertProfile = db.prepare(`
|
|
1581
1638
|
insert into profiles (
|
|
1582
1639
|
id, handle, display_name, bio, followers_count, following_count,
|
|
1583
1640
|
public_metrics_json, avatar_hue, avatar_url, location, url, verified_type,
|
|
@@ -1600,7 +1657,7 @@ export async function importArchive(
|
|
|
1600
1657
|
raw_json = excluded.raw_json,
|
|
1601
1658
|
created_at = excluded.created_at
|
|
1602
1659
|
`);
|
|
1603
|
-
|
|
1660
|
+
const insertProfileIfMissing = db.prepare(`
|
|
1604
1661
|
insert or ignore into profiles (
|
|
1605
1662
|
id, handle, display_name, bio, followers_count, following_count,
|
|
1606
1663
|
public_metrics_json, avatar_hue, avatar_url, location, url, verified_type,
|
|
@@ -1608,7 +1665,7 @@ export async function importArchive(
|
|
|
1608
1665
|
)
|
|
1609
1666
|
values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
1610
1667
|
`);
|
|
1611
|
-
|
|
1668
|
+
const insertTweet = db.prepare(`
|
|
1612
1669
|
insert into tweets (
|
|
1613
1670
|
id, account_id, author_profile_id, kind, text, created_at, is_replied,
|
|
1614
1671
|
reply_to_id, like_count, media_count, bookmarked, liked, entities_json, media_json, quoted_tweet_id
|
|
@@ -1652,14 +1709,16 @@ export async function importArchive(
|
|
|
1652
1709
|
media_json = case when excluded.media_json <> '[]' then excluded.media_json else tweets.media_json end,
|
|
1653
1710
|
quoted_tweet_id = coalesce(excluded.quoted_tweet_id, tweets.quoted_tweet_id)
|
|
1654
1711
|
`);
|
|
1655
|
-
|
|
1656
|
-
|
|
1657
|
-
|
|
1658
|
-
|
|
1659
|
-
|
|
1660
|
-
|
|
1661
|
-
|
|
1662
|
-
|
|
1712
|
+
const deleteTweetFts = db.prepare(
|
|
1713
|
+
"delete from tweets_fts where tweet_id = ?",
|
|
1714
|
+
);
|
|
1715
|
+
const insertTweetFts = db.prepare(
|
|
1716
|
+
"insert into tweets_fts (tweet_id, text) values (?, ?)",
|
|
1717
|
+
);
|
|
1718
|
+
const selectTweetFtsText = db.prepare(
|
|
1719
|
+
"select text from tweets where id = ?",
|
|
1720
|
+
);
|
|
1721
|
+
const insertTimelineEdge = db.prepare(`
|
|
1663
1722
|
insert into tweet_account_edges (
|
|
1664
1723
|
account_id, tweet_id, kind, first_seen_at, last_seen_at, seen_count, source,
|
|
1665
1724
|
raw_json, updated_at
|
|
@@ -1669,7 +1728,7 @@ export async function importArchive(
|
|
|
1669
1728
|
last_seen_at = max(tweet_account_edges.last_seen_at, excluded.last_seen_at),
|
|
1670
1729
|
updated_at = max(tweet_account_edges.updated_at, excluded.updated_at)
|
|
1671
1730
|
`);
|
|
1672
|
-
|
|
1731
|
+
const insertCollection = db.prepare(`
|
|
1673
1732
|
insert into tweet_collections (
|
|
1674
1733
|
account_id, tweet_id, kind, collected_at, source, raw_json, updated_at
|
|
1675
1734
|
) values (?, ?, ?, ?, ?, ?, ?)
|
|
@@ -1685,20 +1744,20 @@ export async function importArchive(
|
|
|
1685
1744
|
end,
|
|
1686
1745
|
updated_at = max(tweet_collections.updated_at, excluded.updated_at)
|
|
1687
1746
|
`);
|
|
1688
|
-
|
|
1747
|
+
const insertConversation = db.prepare(`
|
|
1689
1748
|
insert into dm_conversations (
|
|
1690
1749
|
id, account_id, participant_profile_id, title, last_message_at, unread_count, needs_reply
|
|
1691
1750
|
) values (?, ?, ?, ?, ?, ?, ?)
|
|
1692
1751
|
`);
|
|
1693
|
-
|
|
1752
|
+
const insertMessage = db.prepare(`
|
|
1694
1753
|
insert into dm_messages (
|
|
1695
1754
|
id, conversation_id, sender_profile_id, text, created_at, direction, is_replied, media_count
|
|
1696
1755
|
) values (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1697
1756
|
`);
|
|
1698
|
-
|
|
1699
|
-
|
|
1700
|
-
|
|
1701
|
-
|
|
1757
|
+
const insertDmFts = db.prepare(
|
|
1758
|
+
"insert into dm_fts (message_id, text) values (?, ?)",
|
|
1759
|
+
);
|
|
1760
|
+
const insertFollowSnapshot = db.prepare(`
|
|
1702
1761
|
insert into follow_snapshots (
|
|
1703
1762
|
id, account_id, direction, source, status, page_count, result_count,
|
|
1704
1763
|
started_at, completed_at, raw_meta_json
|
|
@@ -1714,21 +1773,21 @@ export async function importArchive(
|
|
|
1714
1773
|
completed_at = excluded.completed_at,
|
|
1715
1774
|
raw_meta_json = excluded.raw_meta_json
|
|
1716
1775
|
`);
|
|
1717
|
-
|
|
1776
|
+
const insertFollowSnapshotMember = db.prepare(`
|
|
1718
1777
|
insert into follow_snapshot_members (
|
|
1719
1778
|
snapshot_id, profile_id, external_user_id, position
|
|
1720
1779
|
) values (?, ?, ?, ?)
|
|
1721
1780
|
`);
|
|
1722
|
-
|
|
1781
|
+
const selectFollowSnapshotMembers = db.prepare(`
|
|
1723
1782
|
select profile_id, external_user_id
|
|
1724
1783
|
from follow_snapshot_members
|
|
1725
1784
|
where snapshot_id = ?
|
|
1726
1785
|
order by position, profile_id
|
|
1727
1786
|
`);
|
|
1728
|
-
|
|
1729
|
-
|
|
1730
|
-
|
|
1731
|
-
|
|
1787
|
+
const deleteFollowSnapshotMembers = db.prepare(
|
|
1788
|
+
"delete from follow_snapshot_members where snapshot_id = ?",
|
|
1789
|
+
);
|
|
1790
|
+
const deleteArchiveFollowEvents = db.prepare(`
|
|
1732
1791
|
delete from follow_events
|
|
1733
1792
|
where account_id = ? and direction = ? and (
|
|
1734
1793
|
snapshot_id = ? or snapshot_id in (
|
|
@@ -1737,27 +1796,27 @@ export async function importArchive(
|
|
|
1737
1796
|
)
|
|
1738
1797
|
)
|
|
1739
1798
|
`);
|
|
1740
|
-
|
|
1799
|
+
const deleteArchiveFollowSnapshotMembers = db.prepare(`
|
|
1741
1800
|
delete from follow_snapshot_members
|
|
1742
1801
|
where snapshot_id in (
|
|
1743
1802
|
select id from follow_snapshots
|
|
1744
1803
|
where account_id = ? and direction = ? and source = 'archive'
|
|
1745
1804
|
)
|
|
1746
1805
|
`);
|
|
1747
|
-
|
|
1806
|
+
const deleteArchiveFollowSnapshots = db.prepare(`
|
|
1748
1807
|
delete from follow_snapshots
|
|
1749
1808
|
where account_id = ? and direction = ? and source = 'archive'
|
|
1750
1809
|
`);
|
|
1751
|
-
|
|
1810
|
+
const deleteArchiveFollowEdges = db.prepare(`
|
|
1752
1811
|
delete from follow_edges
|
|
1753
1812
|
where account_id = ? and direction = ? and source = 'archive'
|
|
1754
1813
|
`);
|
|
1755
|
-
|
|
1814
|
+
const selectFollowEdges = db.prepare(`
|
|
1756
1815
|
select profile_id, external_user_id, current
|
|
1757
1816
|
from follow_edges
|
|
1758
1817
|
where account_id = ? and direction = ?
|
|
1759
1818
|
`);
|
|
1760
|
-
|
|
1819
|
+
const insertFollowEdge = db.prepare(`
|
|
1761
1820
|
insert into follow_edges (
|
|
1762
1821
|
account_id, direction, profile_id, external_user_id, source, current,
|
|
1763
1822
|
first_seen_at, last_seen_at, ended_at, updated_at
|
|
@@ -1773,25 +1832,25 @@ export async function importArchive(
|
|
|
1773
1832
|
ended_at = null,
|
|
1774
1833
|
updated_at = excluded.updated_at
|
|
1775
1834
|
`);
|
|
1776
|
-
|
|
1835
|
+
const endFollowEdge = db.prepare(`
|
|
1777
1836
|
update follow_edges
|
|
1778
1837
|
set current = 0, ended_at = ?, updated_at = ?
|
|
1779
1838
|
where account_id = ? and direction = ? and profile_id = ?
|
|
1780
1839
|
`);
|
|
1781
|
-
|
|
1840
|
+
const insertFollowEvent = db.prepare(`
|
|
1782
1841
|
insert into follow_events (
|
|
1783
1842
|
id, account_id, direction, profile_id, external_user_id, kind, event_at, snapshot_id
|
|
1784
1843
|
) values (?, ?, ?, ?, ?, ?, ?, ?)
|
|
1785
1844
|
`);
|
|
1786
|
-
|
|
1845
|
+
const clearSelectedLikes = db.prepare(`
|
|
1787
1846
|
delete from tweet_collections
|
|
1788
1847
|
where account_id = ? and kind = 'likes' and source in ('archive', 'legacy')
|
|
1789
1848
|
`);
|
|
1790
|
-
|
|
1849
|
+
const clearSelectedBookmarks = db.prepare(`
|
|
1791
1850
|
delete from tweet_collections
|
|
1792
1851
|
where account_id = ? and kind = 'bookmarks' and source in ('archive', 'legacy')
|
|
1793
1852
|
`);
|
|
1794
|
-
|
|
1853
|
+
const clearTweetLikedFlag = db.prepare(`
|
|
1795
1854
|
update tweets
|
|
1796
1855
|
set liked = 0
|
|
1797
1856
|
where account_id = ?
|
|
@@ -1801,7 +1860,7 @@ export async function importArchive(
|
|
|
1801
1860
|
where account_id = ? and kind = 'likes' and source in ('archive', 'legacy')
|
|
1802
1861
|
)
|
|
1803
1862
|
`);
|
|
1804
|
-
|
|
1863
|
+
const clearTweetBookmarkedFlag = db.prepare(`
|
|
1805
1864
|
update tweets
|
|
1806
1865
|
set bookmarked = 0
|
|
1807
1866
|
where account_id = ?
|
|
@@ -1811,7 +1870,7 @@ export async function importArchive(
|
|
|
1811
1870
|
where account_id = ? and kind = 'bookmarks' and source in ('archive', 'legacy')
|
|
1812
1871
|
)
|
|
1813
1872
|
`);
|
|
1814
|
-
|
|
1873
|
+
const clearSelectedArchiveTweetEdges = db.prepare(`
|
|
1815
1874
|
delete from tweet_account_edges
|
|
1816
1875
|
where account_id = ?
|
|
1817
1876
|
and kind in ('home', 'authored')
|
|
@@ -1829,12 +1888,12 @@ export async function importArchive(
|
|
|
1829
1888
|
)
|
|
1830
1889
|
)
|
|
1831
1890
|
`);
|
|
1832
|
-
|
|
1891
|
+
const deleteOrphanTweetLinkOccurrences = db.prepare(`
|
|
1833
1892
|
delete from link_occurrences
|
|
1834
1893
|
where source_kind = 'tweet'
|
|
1835
1894
|
and source_id not in (select id from tweets)
|
|
1836
1895
|
`);
|
|
1837
|
-
|
|
1896
|
+
const deleteOrphanArchiveCollectionTweets = db.prepare(`
|
|
1838
1897
|
delete from tweets
|
|
1839
1898
|
where account_id = ?
|
|
1840
1899
|
and kind in ('like', 'bookmark')
|
|
@@ -1855,7 +1914,7 @@ export async function importArchive(
|
|
|
1855
1914
|
or referencing_tweet.quoted_tweet_id = tweets.id
|
|
1856
1915
|
)
|
|
1857
1916
|
`);
|
|
1858
|
-
|
|
1917
|
+
const demoteSelectedArchiveTweetsWithCollections = db.prepare(`
|
|
1859
1918
|
update tweets
|
|
1860
1919
|
set kind = case
|
|
1861
1920
|
when exists (
|
|
@@ -1896,7 +1955,7 @@ export async function importArchive(
|
|
|
1896
1955
|
where account_id = ?
|
|
1897
1956
|
)
|
|
1898
1957
|
`);
|
|
1899
|
-
|
|
1958
|
+
const preserveSelectedArchiveTweetsReferencedElsewhere = db.prepare(`
|
|
1900
1959
|
update tweets
|
|
1901
1960
|
set kind = 'archive_stale'
|
|
1902
1961
|
where account_id = ?
|
|
@@ -1973,7 +2032,7 @@ export async function importArchive(
|
|
|
1973
2032
|
)
|
|
1974
2033
|
)
|
|
1975
2034
|
`);
|
|
1976
|
-
|
|
2035
|
+
const deleteSelectedArchiveTweetsWithoutCollections = db.prepare(`
|
|
1977
2036
|
delete from tweets
|
|
1978
2037
|
where account_id = ?
|
|
1979
2038
|
and id in (
|
|
@@ -2046,11 +2105,11 @@ export async function importArchive(
|
|
|
2046
2105
|
)
|
|
2047
2106
|
)
|
|
2048
2107
|
`);
|
|
2049
|
-
|
|
2108
|
+
const deleteOrphanTweetFts = db.prepare(`
|
|
2050
2109
|
delete from tweets_fts
|
|
2051
2110
|
where tweet_id not in (select id from tweets)
|
|
2052
2111
|
`);
|
|
2053
|
-
|
|
2112
|
+
const clearDmFts = db.prepare(`
|
|
2054
2113
|
delete from dm_fts
|
|
2055
2114
|
where message_id in (
|
|
2056
2115
|
select m.id
|
|
@@ -2059,7 +2118,7 @@ export async function importArchive(
|
|
|
2059
2118
|
where c.account_id = ?
|
|
2060
2119
|
)
|
|
2061
2120
|
`);
|
|
2062
|
-
|
|
2121
|
+
const clearDmLinkOccurrences = db.prepare(`
|
|
2063
2122
|
delete from link_occurrences
|
|
2064
2123
|
where source_kind = 'dm'
|
|
2065
2124
|
and source_id in (
|
|
@@ -2069,362 +2128,382 @@ export async function importArchive(
|
|
|
2069
2128
|
where c.account_id = ?
|
|
2070
2129
|
)
|
|
2071
2130
|
`);
|
|
2072
|
-
|
|
2131
|
+
const clearDmMessages = db.prepare(`
|
|
2073
2132
|
delete from dm_messages
|
|
2074
2133
|
where conversation_id in (
|
|
2075
2134
|
select id from dm_conversations where account_id = ?
|
|
2076
2135
|
)
|
|
2077
2136
|
`);
|
|
2078
|
-
|
|
2079
|
-
|
|
2080
|
-
|
|
2137
|
+
const clearDmConversations = db.prepare(
|
|
2138
|
+
"delete from dm_conversations where account_id = ?",
|
|
2139
|
+
);
|
|
2081
2140
|
|
|
2082
|
-
|
|
2083
|
-
|
|
2084
|
-
|
|
2085
|
-
|
|
2086
|
-
|
|
2087
|
-
|
|
2088
|
-
|
|
2089
|
-
|
|
2090
|
-
|
|
2091
|
-
|
|
2141
|
+
function importFollowRows(
|
|
2142
|
+
direction: ArchiveFollowDirection,
|
|
2143
|
+
rows: Array<{ profileId: string; externalUserId: string }>,
|
|
2144
|
+
entryCount: number,
|
|
2145
|
+
now: string,
|
|
2146
|
+
) {
|
|
2147
|
+
const snapshotId = `follow_snapshot_archive_acct_primary_${direction}`;
|
|
2148
|
+
const existingEdges = new Map(
|
|
2149
|
+
(
|
|
2150
|
+
selectFollowEdges.all("acct_primary", direction) as Array<{
|
|
2151
|
+
profile_id: string;
|
|
2152
|
+
external_user_id: string;
|
|
2153
|
+
current: number;
|
|
2154
|
+
}>
|
|
2155
|
+
).map((row) => [row.profile_id, row]),
|
|
2156
|
+
);
|
|
2157
|
+
const existingMemberKey = (
|
|
2158
|
+
selectFollowSnapshotMembers.all(snapshotId) as Array<{
|
|
2092
2159
|
profile_id: string;
|
|
2093
2160
|
external_user_id: string;
|
|
2094
|
-
current: number;
|
|
2095
2161
|
}>
|
|
2096
|
-
).map((row) => [row.profile_id, row]),
|
|
2097
|
-
);
|
|
2098
|
-
const existingMemberKey = (
|
|
2099
|
-
selectFollowSnapshotMembers.all(snapshotId) as Array<{
|
|
2100
|
-
profile_id: string;
|
|
2101
|
-
external_user_id: string;
|
|
2102
|
-
}>
|
|
2103
|
-
)
|
|
2104
|
-
.map(
|
|
2105
|
-
(row, index) =>
|
|
2106
|
-
`${String(index)}:${row.profile_id}:${row.external_user_id}`,
|
|
2107
2162
|
)
|
|
2108
|
-
|
|
2109
|
-
|
|
2110
|
-
|
|
2111
|
-
|
|
2112
|
-
|
|
2113
|
-
|
|
2114
|
-
|
|
2115
|
-
|
|
2116
|
-
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
direction,
|
|
2122
|
-
entryCount,
|
|
2123
|
-
rows.length,
|
|
2124
|
-
now,
|
|
2125
|
-
now,
|
|
2126
|
-
JSON.stringify({ archivePath, result_count: rows.length }),
|
|
2127
|
-
);
|
|
2128
|
-
|
|
2129
|
-
if (membersChanged) {
|
|
2130
|
-
deleteFollowSnapshotMembers.run(snapshotId);
|
|
2131
|
-
}
|
|
2132
|
-
rows.forEach((row, index) => {
|
|
2133
|
-
const profileId = resolveProfileId(row.profileId);
|
|
2134
|
-
currentProfileIds.add(profileId);
|
|
2135
|
-
if (membersChanged) {
|
|
2136
|
-
insertFollowSnapshotMember.run(
|
|
2137
|
-
snapshotId,
|
|
2138
|
-
profileId,
|
|
2139
|
-
row.externalUserId,
|
|
2140
|
-
index,
|
|
2141
|
-
);
|
|
2142
|
-
}
|
|
2163
|
+
.map(
|
|
2164
|
+
(row, index) =>
|
|
2165
|
+
`${String(index)}:${row.profile_id}:${row.external_user_id}`,
|
|
2166
|
+
)
|
|
2167
|
+
.join("\n");
|
|
2168
|
+
const nextMemberKey = rows
|
|
2169
|
+
.map(
|
|
2170
|
+
(row, index) =>
|
|
2171
|
+
`${String(index)}:${row.profileId}:${row.externalUserId}`,
|
|
2172
|
+
)
|
|
2173
|
+
.join("\n");
|
|
2174
|
+
const membersChanged = existingMemberKey !== nextMemberKey;
|
|
2175
|
+
const currentProfileIds = new Set<string>();
|
|
2143
2176
|
|
|
2144
|
-
|
|
2145
|
-
|
|
2177
|
+
insertFollowSnapshot.run(
|
|
2178
|
+
snapshotId,
|
|
2146
2179
|
"acct_primary",
|
|
2147
2180
|
direction,
|
|
2148
|
-
|
|
2149
|
-
|
|
2150
|
-
now,
|
|
2181
|
+
entryCount,
|
|
2182
|
+
rows.length,
|
|
2151
2183
|
now,
|
|
2152
2184
|
now,
|
|
2185
|
+
JSON.stringify({ archivePath, result_count: rows.length }),
|
|
2153
2186
|
);
|
|
2154
|
-
|
|
2187
|
+
|
|
2188
|
+
if (membersChanged) {
|
|
2189
|
+
deleteFollowSnapshotMembers.run(snapshotId);
|
|
2190
|
+
}
|
|
2191
|
+
rows.forEach((row, index) => {
|
|
2192
|
+
const profileId = resolveProfileId(row.profileId);
|
|
2193
|
+
currentProfileIds.add(profileId);
|
|
2194
|
+
if (membersChanged) {
|
|
2195
|
+
insertFollowSnapshotMember.run(
|
|
2196
|
+
snapshotId,
|
|
2197
|
+
profileId,
|
|
2198
|
+
row.externalUserId,
|
|
2199
|
+
index,
|
|
2200
|
+
);
|
|
2201
|
+
}
|
|
2202
|
+
|
|
2203
|
+
const previous = existingEdges.get(profileId);
|
|
2204
|
+
insertFollowEdge.run(
|
|
2205
|
+
"acct_primary",
|
|
2206
|
+
direction,
|
|
2207
|
+
profileId,
|
|
2208
|
+
row.externalUserId,
|
|
2209
|
+
now,
|
|
2210
|
+
now,
|
|
2211
|
+
now,
|
|
2212
|
+
);
|
|
2213
|
+
if (!previous || previous.current === 0) {
|
|
2214
|
+
insertFollowEvent.run(
|
|
2215
|
+
`follow_event_${randomUUID()}`,
|
|
2216
|
+
"acct_primary",
|
|
2217
|
+
direction,
|
|
2218
|
+
profileId,
|
|
2219
|
+
row.externalUserId,
|
|
2220
|
+
"started",
|
|
2221
|
+
now,
|
|
2222
|
+
snapshotId,
|
|
2223
|
+
);
|
|
2224
|
+
}
|
|
2225
|
+
});
|
|
2226
|
+
|
|
2227
|
+
for (const [profileId, previous] of existingEdges) {
|
|
2228
|
+
if (previous.current === 0 || currentProfileIds.has(profileId)) {
|
|
2229
|
+
continue;
|
|
2230
|
+
}
|
|
2231
|
+
endFollowEdge.run(now, now, "acct_primary", direction, profileId);
|
|
2155
2232
|
insertFollowEvent.run(
|
|
2156
2233
|
`follow_event_${randomUUID()}`,
|
|
2157
2234
|
"acct_primary",
|
|
2158
2235
|
direction,
|
|
2159
2236
|
profileId,
|
|
2160
|
-
|
|
2161
|
-
"
|
|
2237
|
+
previous.external_user_id,
|
|
2238
|
+
"ended",
|
|
2162
2239
|
now,
|
|
2163
2240
|
snapshotId,
|
|
2164
2241
|
);
|
|
2165
2242
|
}
|
|
2166
|
-
}
|
|
2243
|
+
}
|
|
2167
2244
|
|
|
2168
|
-
|
|
2169
|
-
|
|
2170
|
-
|
|
2171
|
-
|
|
2172
|
-
|
|
2173
|
-
insertFollowEvent.run(
|
|
2174
|
-
`follow_event_${randomUUID()}`,
|
|
2245
|
+
function clearArchiveFollowRows(direction: ArchiveFollowDirection) {
|
|
2246
|
+
deleteArchiveFollowEvents.run(
|
|
2247
|
+
"acct_primary",
|
|
2248
|
+
direction,
|
|
2249
|
+
`follow_snapshot_archive_acct_primary_${direction}`,
|
|
2175
2250
|
"acct_primary",
|
|
2176
2251
|
direction,
|
|
2177
|
-
profileId,
|
|
2178
|
-
previous.external_user_id,
|
|
2179
|
-
"ended",
|
|
2180
|
-
now,
|
|
2181
|
-
snapshotId,
|
|
2182
2252
|
);
|
|
2253
|
+
deleteArchiveFollowSnapshotMembers.run("acct_primary", direction);
|
|
2254
|
+
deleteArchiveFollowSnapshots.run("acct_primary", direction);
|
|
2255
|
+
deleteArchiveFollowEdges.run("acct_primary", direction);
|
|
2183
2256
|
}
|
|
2184
|
-
}
|
|
2185
2257
|
|
|
2186
|
-
|
|
2187
|
-
|
|
2188
|
-
|
|
2189
|
-
|
|
2190
|
-
`follow_snapshot_archive_acct_primary_${direction}`,
|
|
2191
|
-
"acct_primary",
|
|
2192
|
-
direction,
|
|
2193
|
-
);
|
|
2194
|
-
deleteArchiveFollowSnapshotMembers.run("acct_primary", direction);
|
|
2195
|
-
deleteArchiveFollowSnapshots.run("acct_primary", direction);
|
|
2196
|
-
deleteArchiveFollowEdges.run("acct_primary", direction);
|
|
2197
|
-
}
|
|
2198
|
-
|
|
2199
|
-
db.transaction(() => {
|
|
2200
|
-
if (selection) {
|
|
2201
|
-
if (includeTweets) {
|
|
2202
|
-
clearAuthoredSyncCursors(db, "acct_primary");
|
|
2203
|
-
demoteSelectedArchiveTweetsWithCollections.run(
|
|
2204
|
-
"acct_primary",
|
|
2205
|
-
"acct_primary",
|
|
2206
|
-
"acct_primary",
|
|
2207
|
-
"acct_primary",
|
|
2208
|
-
"acct_primary",
|
|
2209
|
-
localProfile.id,
|
|
2210
|
-
"acct_primary",
|
|
2211
|
-
);
|
|
2212
|
-
preserveSelectedArchiveTweetsReferencedElsewhere.run(
|
|
2213
|
-
"acct_primary",
|
|
2214
|
-
"acct_primary",
|
|
2215
|
-
"acct_primary",
|
|
2216
|
-
localProfile.id,
|
|
2217
|
-
"acct_primary",
|
|
2218
|
-
"acct_primary",
|
|
2219
|
-
"acct_primary",
|
|
2220
|
-
localProfile.id,
|
|
2221
|
-
"acct_primary",
|
|
2222
|
-
"acct_primary",
|
|
2223
|
-
localProfile.id,
|
|
2224
|
-
);
|
|
2225
|
-
deleteSelectedArchiveTweetsWithoutCollections.run(
|
|
2226
|
-
"acct_primary",
|
|
2227
|
-
"acct_primary",
|
|
2228
|
-
"acct_primary",
|
|
2229
|
-
localProfile.id,
|
|
2230
|
-
"acct_primary",
|
|
2231
|
-
"acct_primary",
|
|
2232
|
-
localProfile.id,
|
|
2233
|
-
"acct_primary",
|
|
2234
|
-
"acct_primary",
|
|
2235
|
-
localProfile.id,
|
|
2236
|
-
);
|
|
2237
|
-
deleteOrphanTweetFts.run();
|
|
2238
|
-
deleteOrphanTweetLinkOccurrences.run();
|
|
2239
|
-
clearSelectedArchiveTweetEdges.run(
|
|
2240
|
-
"acct_primary",
|
|
2241
|
-
"acct_primary",
|
|
2242
|
-
localProfile.id,
|
|
2243
|
-
);
|
|
2244
|
-
}
|
|
2245
|
-
if (includeLikes) {
|
|
2246
|
-
clearTweetLikedFlag.run("acct_primary", "acct_primary");
|
|
2247
|
-
clearSelectedLikes.run("acct_primary");
|
|
2248
|
-
}
|
|
2249
|
-
if (includeBookmarks) {
|
|
2250
|
-
clearTweetBookmarkedFlag.run("acct_primary", "acct_primary");
|
|
2251
|
-
clearSelectedBookmarks.run("acct_primary");
|
|
2252
|
-
}
|
|
2253
|
-
if (includeLikes || includeBookmarks) {
|
|
2254
|
-
deleteOrphanArchiveCollectionTweets.run("acct_primary");
|
|
2255
|
-
deleteOrphanTweetFts.run();
|
|
2256
|
-
deleteOrphanTweetLinkOccurrences.run();
|
|
2258
|
+
db.transaction(() => {
|
|
2259
|
+
if (!selection) {
|
|
2260
|
+
clearImportedData(db);
|
|
2261
|
+
clearMentionSyncState(db);
|
|
2257
2262
|
}
|
|
2258
|
-
if (includeDirectMessages) {
|
|
2259
|
-
clearDmLinkOccurrences.run("acct_primary");
|
|
2260
|
-
clearDmFts.run("acct_primary");
|
|
2261
|
-
clearDmMessages.run("acct_primary");
|
|
2262
|
-
clearDmConversations.run("acct_primary");
|
|
2263
|
-
}
|
|
2264
|
-
}
|
|
2265
|
-
|
|
2266
|
-
const writeAccount = selection ? insertAccountIfMissing : insertAccount;
|
|
2267
|
-
writeAccount.run(
|
|
2268
|
-
"acct_primary",
|
|
2269
|
-
accountPayload.displayName,
|
|
2270
|
-
`@${accountPayload.username}`,
|
|
2271
|
-
accountPayload.accountId,
|
|
2272
|
-
"archive",
|
|
2273
|
-
accountPayload.createdAt,
|
|
2274
|
-
);
|
|
2275
2263
|
|
|
2276
|
-
|
|
2277
|
-
|
|
2278
|
-
|
|
2279
|
-
|
|
2280
|
-
|
|
2281
|
-
|
|
2282
|
-
|
|
2283
|
-
|
|
2284
|
-
|
|
2285
|
-
|
|
2286
|
-
|
|
2287
|
-
|
|
2288
|
-
|
|
2289
|
-
|
|
2290
|
-
|
|
2291
|
-
|
|
2292
|
-
|
|
2293
|
-
|
|
2294
|
-
|
|
2295
|
-
|
|
2296
|
-
|
|
2264
|
+
if (selection) {
|
|
2265
|
+
if (includeTweets) {
|
|
2266
|
+
clearAuthoredSyncCursors(db, "acct_primary");
|
|
2267
|
+
demoteSelectedArchiveTweetsWithCollections.run(
|
|
2268
|
+
"acct_primary",
|
|
2269
|
+
"acct_primary",
|
|
2270
|
+
"acct_primary",
|
|
2271
|
+
"acct_primary",
|
|
2272
|
+
"acct_primary",
|
|
2273
|
+
localProfile.id,
|
|
2274
|
+
"acct_primary",
|
|
2275
|
+
);
|
|
2276
|
+
preserveSelectedArchiveTweetsReferencedElsewhere.run(
|
|
2277
|
+
"acct_primary",
|
|
2278
|
+
"acct_primary",
|
|
2279
|
+
"acct_primary",
|
|
2280
|
+
localProfile.id,
|
|
2281
|
+
"acct_primary",
|
|
2282
|
+
"acct_primary",
|
|
2283
|
+
"acct_primary",
|
|
2284
|
+
localProfile.id,
|
|
2285
|
+
"acct_primary",
|
|
2286
|
+
"acct_primary",
|
|
2287
|
+
localProfile.id,
|
|
2288
|
+
);
|
|
2289
|
+
deleteSelectedArchiveTweetsWithoutCollections.run(
|
|
2290
|
+
"acct_primary",
|
|
2291
|
+
"acct_primary",
|
|
2292
|
+
"acct_primary",
|
|
2293
|
+
localProfile.id,
|
|
2294
|
+
"acct_primary",
|
|
2295
|
+
"acct_primary",
|
|
2296
|
+
localProfile.id,
|
|
2297
|
+
"acct_primary",
|
|
2298
|
+
"acct_primary",
|
|
2299
|
+
localProfile.id,
|
|
2300
|
+
);
|
|
2301
|
+
deleteOrphanTweetFts.run();
|
|
2302
|
+
deleteOrphanTweetLinkOccurrences.run();
|
|
2303
|
+
clearSelectedArchiveTweetEdges.run(
|
|
2304
|
+
"acct_primary",
|
|
2305
|
+
"acct_primary",
|
|
2306
|
+
localProfile.id,
|
|
2307
|
+
);
|
|
2308
|
+
}
|
|
2309
|
+
if (includeLikes) {
|
|
2310
|
+
clearTweetLikedFlag.run("acct_primary", "acct_primary");
|
|
2311
|
+
clearSelectedLikes.run("acct_primary");
|
|
2312
|
+
}
|
|
2313
|
+
if (includeBookmarks) {
|
|
2314
|
+
clearTweetBookmarkedFlag.run("acct_primary", "acct_primary");
|
|
2315
|
+
clearSelectedBookmarks.run("acct_primary");
|
|
2316
|
+
}
|
|
2317
|
+
if (includeLikes || includeBookmarks) {
|
|
2318
|
+
deleteOrphanArchiveCollectionTweets.run("acct_primary");
|
|
2319
|
+
deleteOrphanTweetFts.run();
|
|
2320
|
+
deleteOrphanTweetLinkOccurrences.run();
|
|
2321
|
+
}
|
|
2322
|
+
if (includeDirectMessages) {
|
|
2323
|
+
clearDmLinkOccurrences.run("acct_primary");
|
|
2324
|
+
clearDmFts.run("acct_primary");
|
|
2325
|
+
clearDmMessages.run("acct_primary");
|
|
2326
|
+
clearDmConversations.run("acct_primary");
|
|
2327
|
+
}
|
|
2328
|
+
}
|
|
2297
2329
|
|
|
2298
|
-
|
|
2299
|
-
|
|
2300
|
-
tweet.authorProfileId === "profile_me"
|
|
2301
|
-
? localProfile.id
|
|
2302
|
-
: resolveProfileId(tweet.authorProfileId);
|
|
2303
|
-
insertTweet.run(
|
|
2304
|
-
tweet.id,
|
|
2330
|
+
const writeAccount = selection ? insertAccountIfMissing : insertAccount;
|
|
2331
|
+
writeAccount.run(
|
|
2305
2332
|
"acct_primary",
|
|
2306
|
-
|
|
2307
|
-
|
|
2308
|
-
|
|
2309
|
-
|
|
2310
|
-
|
|
2311
|
-
tweet.replyToId,
|
|
2312
|
-
tweet.likeCount,
|
|
2313
|
-
tweet.mediaCount,
|
|
2314
|
-
tweet.bookmarked,
|
|
2315
|
-
tweet.liked,
|
|
2316
|
-
tweet.entitiesJson,
|
|
2317
|
-
tweet.mediaJson,
|
|
2318
|
-
tweet.quotedTweetId,
|
|
2333
|
+
accountPayload.displayName,
|
|
2334
|
+
`@${accountPayload.username}`,
|
|
2335
|
+
accountPayload.accountId,
|
|
2336
|
+
"archive",
|
|
2337
|
+
accountPayload.createdAt,
|
|
2319
2338
|
);
|
|
2320
|
-
|
|
2321
|
-
|
|
2322
|
-
|
|
2323
|
-
|
|
2339
|
+
|
|
2340
|
+
const writeProfile =
|
|
2341
|
+
!selection || includeProfiles ? insertProfile : insertProfileIfMissing;
|
|
2342
|
+
for (const profile of profiles.values()) {
|
|
2343
|
+
writeProfile.run(
|
|
2344
|
+
profile.id,
|
|
2345
|
+
profile.handle,
|
|
2346
|
+
profile.displayName,
|
|
2347
|
+
profile.bio,
|
|
2348
|
+
profile.followersCount,
|
|
2349
|
+
profile.followingCount,
|
|
2350
|
+
profile.publicMetricsJson,
|
|
2351
|
+
profile.avatarHue,
|
|
2352
|
+
profile.avatarUrl,
|
|
2353
|
+
profile.location,
|
|
2354
|
+
profile.url,
|
|
2355
|
+
profile.verifiedType,
|
|
2356
|
+
profile.entitiesJson,
|
|
2357
|
+
profile.rawJson,
|
|
2358
|
+
profile.createdAt,
|
|
2359
|
+
);
|
|
2360
|
+
}
|
|
2361
|
+
|
|
2362
|
+
for (const tweet of tweetRows) {
|
|
2363
|
+
const authorProfileId =
|
|
2364
|
+
tweet.authorProfileId === "profile_me"
|
|
2365
|
+
? localProfile.id
|
|
2366
|
+
: resolveProfileId(tweet.authorProfileId);
|
|
2367
|
+
insertTweet.run(
|
|
2324
2368
|
tweet.id,
|
|
2369
|
+
"acct_primary",
|
|
2370
|
+
authorProfileId,
|
|
2325
2371
|
tweet.kind,
|
|
2372
|
+
tweet.text,
|
|
2326
2373
|
tweet.createdAt,
|
|
2327
|
-
tweet.
|
|
2328
|
-
|
|
2374
|
+
tweet.isReplied,
|
|
2375
|
+
tweet.replyToId,
|
|
2376
|
+
tweet.likeCount,
|
|
2377
|
+
tweet.mediaCount,
|
|
2378
|
+
tweet.bookmarked,
|
|
2379
|
+
tweet.liked,
|
|
2380
|
+
tweet.entitiesJson,
|
|
2381
|
+
tweet.mediaJson,
|
|
2382
|
+
tweet.quotedTweetId,
|
|
2329
2383
|
);
|
|
2384
|
+
deleteTweetFts.run(tweet.id);
|
|
2385
|
+
if (tweet.kind === "home") {
|
|
2386
|
+
insertTimelineEdge.run(
|
|
2387
|
+
"acct_primary",
|
|
2388
|
+
tweet.id,
|
|
2389
|
+
tweet.kind,
|
|
2390
|
+
tweet.createdAt,
|
|
2391
|
+
tweet.createdAt,
|
|
2392
|
+
new Date().toISOString(),
|
|
2393
|
+
);
|
|
2394
|
+
}
|
|
2395
|
+
if (authorProfileId === localProfile.id) {
|
|
2396
|
+
insertTimelineEdge.run(
|
|
2397
|
+
"acct_primary",
|
|
2398
|
+
tweet.id,
|
|
2399
|
+
"authored",
|
|
2400
|
+
tweet.createdAt,
|
|
2401
|
+
tweet.createdAt,
|
|
2402
|
+
new Date().toISOString(),
|
|
2403
|
+
);
|
|
2404
|
+
}
|
|
2405
|
+
const storedTweet = selectTweetFtsText.get(tweet.id) as
|
|
2406
|
+
| { text: string }
|
|
2407
|
+
| undefined;
|
|
2408
|
+
insertTweetFts.run(tweet.id, storedTweet?.text ?? tweet.text);
|
|
2330
2409
|
}
|
|
2331
|
-
|
|
2332
|
-
|
|
2410
|
+
|
|
2411
|
+
const importedAt = new Date().toISOString();
|
|
2412
|
+
for (const collection of collectionRows) {
|
|
2413
|
+
insertCollection.run(
|
|
2333
2414
|
"acct_primary",
|
|
2334
|
-
|
|
2335
|
-
|
|
2336
|
-
|
|
2337
|
-
|
|
2338
|
-
|
|
2415
|
+
collection.tweetId,
|
|
2416
|
+
collection.kind,
|
|
2417
|
+
collection.collectedAt,
|
|
2418
|
+
collection.source,
|
|
2419
|
+
collection.rawJson,
|
|
2420
|
+
importedAt,
|
|
2339
2421
|
);
|
|
2340
2422
|
}
|
|
2341
|
-
const storedTweet = selectTweetFtsText.get(tweet.id) as
|
|
2342
|
-
| { text: string }
|
|
2343
|
-
| undefined;
|
|
2344
|
-
insertTweetFts.run(tweet.id, storedTweet?.text ?? tweet.text);
|
|
2345
|
-
}
|
|
2346
2423
|
|
|
2347
|
-
|
|
2348
|
-
|
|
2349
|
-
|
|
2350
|
-
|
|
2351
|
-
|
|
2352
|
-
|
|
2353
|
-
|
|
2354
|
-
|
|
2355
|
-
|
|
2356
|
-
|
|
2357
|
-
|
|
2358
|
-
}
|
|
2424
|
+
for (const conversation of conversations.values()) {
|
|
2425
|
+
insertConversation.run(
|
|
2426
|
+
conversation.id,
|
|
2427
|
+
conversation.accountId,
|
|
2428
|
+
conversation.participantProfileId,
|
|
2429
|
+
conversation.title,
|
|
2430
|
+
conversation.lastMessageAt,
|
|
2431
|
+
conversation.unreadCount,
|
|
2432
|
+
conversation.needsReply,
|
|
2433
|
+
);
|
|
2434
|
+
}
|
|
2359
2435
|
|
|
2360
|
-
|
|
2361
|
-
|
|
2362
|
-
|
|
2363
|
-
|
|
2364
|
-
|
|
2365
|
-
|
|
2366
|
-
|
|
2367
|
-
|
|
2368
|
-
|
|
2369
|
-
|
|
2370
|
-
|
|
2436
|
+
for (const message of dmMessages) {
|
|
2437
|
+
insertMessage.run(
|
|
2438
|
+
message.id,
|
|
2439
|
+
message.conversationId,
|
|
2440
|
+
message.senderProfileId,
|
|
2441
|
+
message.text,
|
|
2442
|
+
message.createdAt,
|
|
2443
|
+
message.direction,
|
|
2444
|
+
message.direction === "outbound" ? 1 : 0,
|
|
2445
|
+
message.mediaCount,
|
|
2446
|
+
);
|
|
2447
|
+
insertDmFts.run(message.id, message.text);
|
|
2448
|
+
}
|
|
2371
2449
|
|
|
2372
|
-
|
|
2373
|
-
|
|
2374
|
-
|
|
2375
|
-
|
|
2376
|
-
|
|
2377
|
-
|
|
2378
|
-
|
|
2379
|
-
|
|
2380
|
-
|
|
2381
|
-
|
|
2382
|
-
)
|
|
2383
|
-
|
|
2384
|
-
|
|
2450
|
+
if (includeFollowers && followerEntries.length > 0) {
|
|
2451
|
+
importFollowRows(
|
|
2452
|
+
"followers",
|
|
2453
|
+
followerRows,
|
|
2454
|
+
followerEntries.length,
|
|
2455
|
+
importedAt,
|
|
2456
|
+
);
|
|
2457
|
+
} else if (includeFollowers) {
|
|
2458
|
+
clearArchiveFollowRows("followers");
|
|
2459
|
+
}
|
|
2460
|
+
if (includeFollowing && followingEntries.length > 0) {
|
|
2461
|
+
importFollowRows(
|
|
2462
|
+
"following",
|
|
2463
|
+
followingRows,
|
|
2464
|
+
followingEntries.length,
|
|
2465
|
+
importedAt,
|
|
2466
|
+
);
|
|
2467
|
+
} else if (includeFollowing) {
|
|
2468
|
+
clearArchiveFollowRows("following");
|
|
2469
|
+
}
|
|
2470
|
+
})();
|
|
2385
2471
|
|
|
2386
|
-
|
|
2387
|
-
|
|
2388
|
-
|
|
2389
|
-
|
|
2390
|
-
|
|
2391
|
-
|
|
2392
|
-
|
|
2393
|
-
|
|
2394
|
-
|
|
2395
|
-
|
|
2396
|
-
|
|
2397
|
-
|
|
2398
|
-
|
|
2399
|
-
|
|
2400
|
-
|
|
2401
|
-
|
|
2402
|
-
|
|
2403
|
-
|
|
2404
|
-
|
|
2405
|
-
}
|
|
2406
|
-
})
|
|
2472
|
+
return {
|
|
2473
|
+
ok: true,
|
|
2474
|
+
archivePath,
|
|
2475
|
+
account: {
|
|
2476
|
+
id: accountPayload.accountId,
|
|
2477
|
+
handle: accountPayload.username,
|
|
2478
|
+
displayName: accountPayload.displayName,
|
|
2479
|
+
},
|
|
2480
|
+
counts: {
|
|
2481
|
+
tweets: authoredTweetCount,
|
|
2482
|
+
likes: likeCount,
|
|
2483
|
+
bookmarks: bookmarkCount,
|
|
2484
|
+
dmConversations: conversations.size,
|
|
2485
|
+
dmMessages: dmMessages.length,
|
|
2486
|
+
profiles: profiles.size,
|
|
2487
|
+
mediaFiles: mediaFileCounts,
|
|
2488
|
+
followers: followerRows.length,
|
|
2489
|
+
following: followingRows.length,
|
|
2490
|
+
},
|
|
2491
|
+
};
|
|
2492
|
+
});
|
|
2493
|
+
}
|
|
2407
2494
|
|
|
2408
|
-
|
|
2409
|
-
|
|
2410
|
-
|
|
2411
|
-
|
|
2412
|
-
|
|
2413
|
-
|
|
2414
|
-
|
|
2415
|
-
|
|
2416
|
-
|
|
2417
|
-
|
|
2418
|
-
|
|
2419
|
-
|
|
2420
|
-
dmConversations: conversations.size,
|
|
2421
|
-
dmMessages: dmMessages.length,
|
|
2422
|
-
profiles: profiles.size,
|
|
2423
|
-
mediaFiles: mediaFileCounts,
|
|
2424
|
-
followers: followerRows.length,
|
|
2425
|
-
following: followingRows.length,
|
|
2426
|
-
},
|
|
2427
|
-
};
|
|
2495
|
+
export function importArchiveEffect(
|
|
2496
|
+
archivePath: string,
|
|
2497
|
+
options: ImportArchiveOptions = {},
|
|
2498
|
+
): Effect.Effect<ImportedArchiveSummary, unknown> {
|
|
2499
|
+
return importArchiveInternalEffect(archivePath, options);
|
|
2500
|
+
}
|
|
2501
|
+
|
|
2502
|
+
export function importArchive(
|
|
2503
|
+
archivePath: string,
|
|
2504
|
+
options: ImportArchiveOptions = {},
|
|
2505
|
+
): Promise<ImportedArchiveSummary> {
|
|
2506
|
+
return runEffectPromise(importArchiveEffect(archivePath, options));
|
|
2428
2507
|
}
|
|
2429
2508
|
|
|
2430
2509
|
export const __test__ = {
|