birdclaw 0.5.0 → 0.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +63 -0
- package/README.md +55 -5
- package/bin/birdclaw.mjs +50 -11
- package/package.json +9 -7
- package/public/birdclaw-mark.png +0 -0
- package/scripts/browser-perf.mjs +400 -0
- package/scripts/build-docs-site.mjs +940 -0
- package/scripts/docs-site-assets.mjs +311 -0
- package/scripts/run-vitest.mjs +21 -0
- package/scripts/sanitize-node-options.mjs +23 -0
- package/scripts/start-test-server.mjs +42 -0
- package/src/cli.ts +422 -14
- package/src/components/AccountSwitcher.tsx +186 -0
- package/src/components/AppNav.tsx +29 -9
- package/src/components/AvatarChip.tsx +9 -3
- package/src/components/BrandMark.tsx +67 -0
- package/src/components/ConversationThread.tsx +11 -10
- package/src/components/DmWorkspace.tsx +39 -14
- package/src/components/FeedState.tsx +147 -0
- package/src/components/InboxCard.tsx +14 -2
- package/src/components/LinkPreviewCard.tsx +40 -18
- package/src/components/MarkdownViewer.tsx +452 -0
- package/src/components/SavedTimelineView.tsx +64 -56
- package/src/components/SyncNowButton.tsx +137 -0
- package/src/components/ThemeSlider.tsx +49 -93
- package/src/components/TimelineCard.tsx +364 -136
- package/src/components/TimelineRouteFrame.tsx +170 -0
- package/src/components/TweetMediaGrid.tsx +170 -24
- package/src/components/TweetRichText.tsx +28 -13
- package/src/components/account-selection.ts +64 -0
- package/src/components/useTimelineRouteData.ts +153 -0
- package/src/lib/account-sync-job.ts +654 -0
- package/src/lib/actions-transport.ts +216 -146
- package/src/lib/api-client.ts +304 -0
- package/src/lib/archive-finder.ts +72 -53
- package/src/lib/archive-import.ts +1377 -1298
- package/src/lib/authored-live.ts +261 -204
- package/src/lib/avatar-cache.ts +159 -44
- package/src/lib/backup.ts +1532 -951
- package/src/lib/bird-actions.ts +139 -57
- package/src/lib/bird-command.ts +101 -28
- package/src/lib/bird.ts +549 -194
- package/src/lib/blocklist.ts +40 -23
- package/src/lib/blocks-write.ts +129 -80
- package/src/lib/blocks.ts +165 -97
- package/src/lib/bookmark-sync-job.ts +250 -160
- package/src/lib/conversation-surface.ts +205 -0
- package/src/lib/db.ts +35 -3
- package/src/lib/dms-live.ts +720 -66
- package/src/lib/effect-runtime.ts +45 -0
- package/src/lib/follow-graph.ts +224 -180
- package/src/lib/http-effect.ts +222 -0
- package/src/lib/inbox.ts +74 -43
- package/src/lib/link-index.ts +88 -76
- package/src/lib/link-insights.ts +24 -0
- package/src/lib/link-preview-metadata.ts +472 -52
- package/src/lib/media-fetch.ts +286 -213
- package/src/lib/mention-threads-live.ts +352 -288
- package/src/lib/mentions-live.ts +390 -342
- package/src/lib/moderation-target.ts +102 -65
- package/src/lib/moderation-write.ts +77 -18
- package/src/lib/mutes-write.ts +129 -80
- package/src/lib/mutes.ts +8 -1
- package/src/lib/openai.ts +84 -53
- package/src/lib/period-digest.ts +953 -0
- package/src/lib/profile-affiliation-hydration.ts +93 -54
- package/src/lib/profile-hydration.ts +124 -72
- package/src/lib/profile-replies.ts +60 -43
- package/src/lib/profile-resolver.ts +402 -294
- package/src/lib/queries.ts +1024 -189
- package/src/lib/research.ts +165 -120
- package/src/lib/sqlite.ts +1 -0
- package/src/lib/timeline-collections-live.ts +204 -167
- package/src/lib/timeline-live.ts +60 -39
- package/src/lib/tweet-lookup.ts +30 -19
- package/src/lib/types.ts +38 -1
- package/src/lib/ui.ts +41 -10
- package/src/lib/url-expansion.ts +226 -55
- package/src/lib/url-safety.ts +220 -0
- package/src/lib/web-sync.ts +511 -0
- package/src/lib/whois.ts +166 -147
- package/src/lib/x-web.ts +102 -71
- package/src/lib/xurl.ts +681 -411
- package/src/routeTree.gen.ts +63 -0
- package/src/routes/__root.tsx +25 -5
- package/src/routes/api/action.tsx +127 -78
- package/src/routes/api/avatar.tsx +39 -30
- package/src/routes/api/blocks.tsx +26 -23
- package/src/routes/api/conversation.tsx +25 -14
- package/src/routes/api/inbox.tsx +27 -21
- package/src/routes/api/link-insights.tsx +31 -25
- package/src/routes/api/link-preview.tsx +25 -21
- package/src/routes/api/period-digest.tsx +123 -0
- package/src/routes/api/profile-hydrate.tsx +31 -28
- package/src/routes/api/query.tsx +79 -55
- package/src/routes/api/status.tsx +18 -10
- package/src/routes/api/sync.tsx +105 -0
- package/src/routes/dms.tsx +195 -55
- package/src/routes/inbox.tsx +32 -19
- package/src/routes/index.tsx +21 -127
- package/src/routes/links.tsx +50 -5
- package/src/routes/mentions.tsx +21 -127
- package/src/routes/today.tsx +441 -0
- package/src/styles.css +74 -11
- package/vite.config.ts +8 -0
package/src/lib/mentions-live.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
import type { Database } from "./sqlite";
|
|
2
|
-
import {
|
|
2
|
+
import { Effect } from "effect";
|
|
3
|
+
import { listMentionsViaBirdEffect } from "./bird";
|
|
3
4
|
import type { MentionsDataSource } from "./config";
|
|
4
5
|
import { getNativeDb } from "./db";
|
|
6
|
+
import { runEffectPromise, tryPromise } from "./effect-runtime";
|
|
5
7
|
import { buildMediaJsonFromIncludes, countTweetMedia } from "./media-includes";
|
|
6
8
|
import { serializeMentionItemsAsXurlCompatible } from "./mentions-export";
|
|
7
9
|
import { listTimelineItems } from "./queries";
|
|
@@ -22,6 +24,27 @@ export const DEFAULT_MENTIONS_CACHE_TTL_MS = 2 * 60_000;
|
|
|
22
24
|
const MIN_XURL_MENTIONS_LIMIT = 5;
|
|
23
25
|
const MAX_XURL_MENTIONS_LIMIT = 100;
|
|
24
26
|
type MentionSyncMode = Exclude<MentionsDataSource, "birdclaw">;
|
|
27
|
+
export interface SyncMentionsOptions {
|
|
28
|
+
account?: string;
|
|
29
|
+
mode?: string;
|
|
30
|
+
limit?: number;
|
|
31
|
+
maxPages?: number;
|
|
32
|
+
refresh?: boolean;
|
|
33
|
+
cacheTtlMs?: number;
|
|
34
|
+
sinceId?: string;
|
|
35
|
+
startTime?: string;
|
|
36
|
+
}
|
|
37
|
+
interface ExportMentionsViaCachedLiveSourceOptions {
|
|
38
|
+
mode: MentionsDataSource;
|
|
39
|
+
account?: string;
|
|
40
|
+
search?: string;
|
|
41
|
+
replyFilter?: ReplyFilter;
|
|
42
|
+
limit?: number;
|
|
43
|
+
all?: boolean;
|
|
44
|
+
maxPages?: number;
|
|
45
|
+
refresh?: boolean;
|
|
46
|
+
cacheTtlMs?: number;
|
|
47
|
+
}
|
|
25
48
|
type MentionScanBoundary =
|
|
26
49
|
| { kind: "auto" }
|
|
27
50
|
| { kind: "since"; sinceId: string }
|
|
@@ -44,6 +67,17 @@ interface MentionHighWaterValue {
|
|
|
44
67
|
sinceId: string;
|
|
45
68
|
}
|
|
46
69
|
|
|
70
|
+
function toError(error: unknown) {
|
|
71
|
+
return error instanceof Error ? error : new Error(String(error));
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
function trySync<T>(try_: () => T) {
|
|
75
|
+
return Effect.try({
|
|
76
|
+
try: try_,
|
|
77
|
+
catch: toError,
|
|
78
|
+
});
|
|
79
|
+
}
|
|
80
|
+
|
|
47
81
|
function getMentionsFetchModeKey({
|
|
48
82
|
scope,
|
|
49
83
|
mode,
|
|
@@ -655,7 +689,7 @@ function mergeMentionPayloads(
|
|
|
655
689
|
};
|
|
656
690
|
}
|
|
657
691
|
|
|
658
|
-
|
|
692
|
+
function fetchMentionsViaXurlEffect({
|
|
659
693
|
resolvedAccount,
|
|
660
694
|
limit,
|
|
661
695
|
all,
|
|
@@ -672,43 +706,51 @@ async function fetchMentionsViaXurl({
|
|
|
672
706
|
startPaginationToken?: string;
|
|
673
707
|
startTime?: string;
|
|
674
708
|
}) {
|
|
675
|
-
|
|
676
|
-
|
|
677
|
-
|
|
678
|
-
`Could not resolve Twitter user id for @${resolvedAccount.username}`,
|
|
709
|
+
return Effect.gen(function* () {
|
|
710
|
+
const [accountUser] = yield* tryPromise(() =>
|
|
711
|
+
lookupUsersByHandles([resolvedAccount.username]),
|
|
679
712
|
);
|
|
680
|
-
|
|
713
|
+
if (!accountUser?.id) {
|
|
714
|
+
return yield* Effect.fail(
|
|
715
|
+
new Error(
|
|
716
|
+
`Could not resolve Twitter user id for @${resolvedAccount.username}`,
|
|
717
|
+
),
|
|
718
|
+
);
|
|
719
|
+
}
|
|
681
720
|
|
|
682
|
-
|
|
683
|
-
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
687
|
-
|
|
688
|
-
|
|
689
|
-
|
|
690
|
-
|
|
691
|
-
|
|
692
|
-
|
|
693
|
-
|
|
694
|
-
|
|
695
|
-
|
|
696
|
-
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
700
|
-
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
|
|
705
|
-
|
|
721
|
+
const pages: XurlMentionsResponse[] = [];
|
|
722
|
+
let nextToken: string | undefined = startPaginationToken;
|
|
723
|
+
let pageCount = 0;
|
|
724
|
+
do {
|
|
725
|
+
const payload = yield* tryPromise(() =>
|
|
726
|
+
listMentionsViaXurl({
|
|
727
|
+
maxResults: limit,
|
|
728
|
+
username: resolvedAccount.username,
|
|
729
|
+
userId: String(accountUser.id),
|
|
730
|
+
paginationToken: nextToken,
|
|
731
|
+
...(sinceId ? { sinceId } : {}),
|
|
732
|
+
...(startTime ? { startTime } : {}),
|
|
733
|
+
}),
|
|
734
|
+
);
|
|
735
|
+
pages.push(payload);
|
|
736
|
+
const metaNextToken =
|
|
737
|
+
typeof payload.meta?.next_token === "string"
|
|
738
|
+
? payload.meta.next_token
|
|
739
|
+
: undefined;
|
|
740
|
+
nextToken = metaNextToken;
|
|
741
|
+
pageCount += 1;
|
|
742
|
+
} while (
|
|
743
|
+
all &&
|
|
744
|
+
nextToken &&
|
|
745
|
+
(parsedMaxPages === null || pageCount < parsedMaxPages)
|
|
746
|
+
);
|
|
706
747
|
|
|
707
|
-
|
|
748
|
+
return mergeMentionPayloads(pages);
|
|
749
|
+
});
|
|
708
750
|
}
|
|
709
751
|
|
|
710
|
-
|
|
711
|
-
return
|
|
752
|
+
function fetchMentionsViaBirdEffect({ limit }: { limit: number }) {
|
|
753
|
+
return listMentionsViaBirdEffect({ maxResults: limit });
|
|
712
754
|
}
|
|
713
755
|
|
|
714
756
|
function isMaxPagesPartial({
|
|
@@ -725,7 +767,7 @@ function isMaxPagesPartial({
|
|
|
725
767
|
);
|
|
726
768
|
}
|
|
727
769
|
|
|
728
|
-
export
|
|
770
|
+
export function syncMentionsEffect({
|
|
729
771
|
account,
|
|
730
772
|
mode,
|
|
731
773
|
limit = 20,
|
|
@@ -734,222 +776,239 @@ export async function syncMentions({
|
|
|
734
776
|
cacheTtlMs,
|
|
735
777
|
sinceId,
|
|
736
778
|
startTime,
|
|
737
|
-
}: {
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
}
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
|
|
754
|
-
|
|
755
|
-
|
|
756
|
-
|
|
757
|
-
|
|
758
|
-
|
|
759
|
-
const fetchAll = parsedMode === "xurl" && parsedMaxPages !== null;
|
|
760
|
-
const db = getNativeDb();
|
|
761
|
-
const resolvedAccount = resolveAccount(db, account);
|
|
762
|
-
const cursorShape: MentionScanShape = {
|
|
763
|
-
endpoint: "mentions",
|
|
764
|
-
mode: parsedMode,
|
|
765
|
-
accountId: resolvedAccount.accountId,
|
|
766
|
-
pageSize: limit,
|
|
767
|
-
boundary: getMentionCursorBoundary({
|
|
768
|
-
explicitSinceId,
|
|
769
|
-
explicitStartTime,
|
|
770
|
-
}),
|
|
771
|
-
};
|
|
772
|
-
const cursorKey = getMentionCursorKey(cursorShape);
|
|
773
|
-
const legacyCursorKeys = getLegacyMentionCursorKeys(cursorShape);
|
|
774
|
-
const cursor =
|
|
775
|
-
parsedMode === "xurl"
|
|
776
|
-
? readMentionCursor({
|
|
777
|
-
db,
|
|
778
|
-
shape: cursorShape,
|
|
779
|
-
cursorKey,
|
|
780
|
-
legacyCursorKeys,
|
|
781
|
-
})
|
|
782
|
-
: undefined;
|
|
783
|
-
const startPaginationToken = cursor?.token;
|
|
784
|
-
const cursorSinceId =
|
|
785
|
-
cursor?.boundary?.kind === "since" ? cursor.boundary.sinceId : undefined;
|
|
786
|
-
const cursorStartTime =
|
|
787
|
-
cursor?.boundary?.kind === "start" ? cursor.boundary.startTime : undefined;
|
|
788
|
-
const committedSinceId =
|
|
789
|
-
parsedMode === "xurl" &&
|
|
790
|
-
cursorShape.boundary.kind === "auto" &&
|
|
791
|
-
!startPaginationToken
|
|
792
|
-
? readMentionHighWaterId(db, parsedMode, resolvedAccount.accountId)
|
|
793
|
-
: undefined;
|
|
794
|
-
const seededSinceId =
|
|
795
|
-
parsedMode === "xurl" &&
|
|
796
|
-
!explicitSinceId &&
|
|
797
|
-
!explicitStartTime &&
|
|
798
|
-
!startPaginationToken
|
|
799
|
-
? (committedSinceId ??
|
|
800
|
-
findNewestArchiveMentionId(db, resolvedAccount.accountId))
|
|
801
|
-
: undefined;
|
|
802
|
-
const resolvedSinceId = startPaginationToken
|
|
803
|
-
? cursorSinceId
|
|
804
|
-
: (explicitSinceId ?? seededSinceId);
|
|
805
|
-
const resolvedStartTime = startPaginationToken
|
|
806
|
-
? cursorStartTime
|
|
807
|
-
: !resolvedSinceId
|
|
808
|
-
? explicitStartTime
|
|
809
|
-
: undefined;
|
|
810
|
-
const resultShape: MentionScanShape = {
|
|
811
|
-
endpoint: "mentions",
|
|
812
|
-
mode: parsedMode,
|
|
813
|
-
accountId: resolvedAccount.accountId,
|
|
814
|
-
pageSize: limit,
|
|
815
|
-
boundary: getMentionRequestBoundary({
|
|
816
|
-
sinceId: resolvedSinceId,
|
|
817
|
-
startTime: resolvedStartTime,
|
|
818
|
-
}),
|
|
819
|
-
};
|
|
820
|
-
const resultCacheKey = getMentionResultCacheKey({
|
|
821
|
-
shape: resultShape,
|
|
822
|
-
all: fetchAll,
|
|
823
|
-
maxPages: parsedMaxPages,
|
|
824
|
-
});
|
|
825
|
-
const ttlMs = parseCacheTtlMs(cacheTtlMs);
|
|
826
|
-
const cached = startPaginationToken
|
|
827
|
-
? null
|
|
828
|
-
: readSyncCache<XurlMentionsResponse>(resultCacheKey, db);
|
|
829
|
-
const cachedPaginationToken = getCachedPaginationToken(cached);
|
|
830
|
-
const cacheAgeMs = cached
|
|
831
|
-
? Date.now() - new Date(cached.updatedAt).getTime()
|
|
832
|
-
: Number.POSITIVE_INFINITY;
|
|
833
|
-
|
|
834
|
-
if (
|
|
835
|
-
!startPaginationToken &&
|
|
836
|
-
!cachedPaginationToken &&
|
|
837
|
-
!refresh &&
|
|
838
|
-
cached &&
|
|
839
|
-
cacheAgeMs <= ttlMs
|
|
840
|
-
) {
|
|
841
|
-
mergeMentionsIntoLocalStore(
|
|
842
|
-
db,
|
|
843
|
-
resolvedAccount.accountId,
|
|
844
|
-
cached.value,
|
|
845
|
-
parsedMode,
|
|
846
|
-
);
|
|
847
|
-
return {
|
|
848
|
-
ok: true,
|
|
849
|
-
source: "cache",
|
|
850
|
-
kind: "mentions",
|
|
779
|
+
}: SyncMentionsOptions) {
|
|
780
|
+
return Effect.gen(function* () {
|
|
781
|
+
const parsedMode = yield* trySync(() => parseSyncMode(mode));
|
|
782
|
+
const explicitSinceId = sinceId?.trim() || undefined;
|
|
783
|
+
const explicitStartTime = startTime?.trim() || undefined;
|
|
784
|
+
if (parsedMode === "bird" && (explicitSinceId || explicitStartTime)) {
|
|
785
|
+
return yield* Effect.fail(
|
|
786
|
+
new Error("bird mode does not support --since-id or --start-time"),
|
|
787
|
+
);
|
|
788
|
+
}
|
|
789
|
+
if (parsedMode === "xurl") {
|
|
790
|
+
yield* trySync(() => assertXurlLimit(limit));
|
|
791
|
+
} else {
|
|
792
|
+
yield* trySync(() => assertBirdLimit(limit));
|
|
793
|
+
}
|
|
794
|
+
const parsedMaxPages = yield* trySync(() => parseMaxPages(maxPages));
|
|
795
|
+
const fetchAll = parsedMode === "xurl" && parsedMaxPages !== null;
|
|
796
|
+
const db = yield* trySync(() => getNativeDb());
|
|
797
|
+
const resolvedAccount = yield* trySync(() => resolveAccount(db, account));
|
|
798
|
+
const cursorShape: MentionScanShape = {
|
|
799
|
+
endpoint: "mentions",
|
|
800
|
+
mode: parsedMode,
|
|
851
801
|
accountId: resolvedAccount.accountId,
|
|
852
|
-
|
|
853
|
-
|
|
854
|
-
|
|
855
|
-
|
|
802
|
+
pageSize: limit,
|
|
803
|
+
boundary: getMentionCursorBoundary({
|
|
804
|
+
explicitSinceId,
|
|
805
|
+
explicitStartTime,
|
|
856
806
|
}),
|
|
857
|
-
payload: cached.value,
|
|
858
807
|
};
|
|
859
|
-
|
|
860
|
-
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
|
|
869
|
-
|
|
870
|
-
|
|
871
|
-
|
|
808
|
+
const cursorKey = getMentionCursorKey(cursorShape);
|
|
809
|
+
const legacyCursorKeys = getLegacyMentionCursorKeys(cursorShape);
|
|
810
|
+
const cursor =
|
|
811
|
+
parsedMode === "xurl"
|
|
812
|
+
? yield* trySync(() =>
|
|
813
|
+
readMentionCursor({
|
|
814
|
+
db,
|
|
815
|
+
shape: cursorShape,
|
|
816
|
+
cursorKey,
|
|
817
|
+
legacyCursorKeys,
|
|
818
|
+
}),
|
|
819
|
+
)
|
|
820
|
+
: undefined;
|
|
821
|
+
const startPaginationToken = cursor?.token;
|
|
822
|
+
const cursorSinceId =
|
|
823
|
+
cursor?.boundary?.kind === "since" ? cursor.boundary.sinceId : undefined;
|
|
824
|
+
const cursorStartTime =
|
|
825
|
+
cursor?.boundary?.kind === "start"
|
|
826
|
+
? cursor.boundary.startTime
|
|
827
|
+
: undefined;
|
|
828
|
+
const committedSinceId =
|
|
829
|
+
parsedMode === "xurl" &&
|
|
830
|
+
cursorShape.boundary.kind === "auto" &&
|
|
831
|
+
!startPaginationToken
|
|
832
|
+
? yield* trySync(() =>
|
|
833
|
+
readMentionHighWaterId(db, parsedMode, resolvedAccount.accountId),
|
|
834
|
+
)
|
|
835
|
+
: undefined;
|
|
836
|
+
const seededSinceId =
|
|
837
|
+
parsedMode === "xurl" &&
|
|
838
|
+
!explicitSinceId &&
|
|
839
|
+
!explicitStartTime &&
|
|
840
|
+
!startPaginationToken
|
|
841
|
+
? (committedSinceId ??
|
|
842
|
+
(yield* trySync(() =>
|
|
843
|
+
findNewestArchiveMentionId(db, resolvedAccount.accountId),
|
|
844
|
+
)))
|
|
845
|
+
: undefined;
|
|
846
|
+
const resolvedSinceId = startPaginationToken
|
|
847
|
+
? cursorSinceId
|
|
848
|
+
: (explicitSinceId ?? seededSinceId);
|
|
849
|
+
const resolvedStartTime = startPaginationToken
|
|
850
|
+
? cursorStartTime
|
|
851
|
+
: !resolvedSinceId
|
|
852
|
+
? explicitStartTime
|
|
853
|
+
: undefined;
|
|
854
|
+
const resultShape: MentionScanShape = {
|
|
855
|
+
endpoint: "mentions",
|
|
856
|
+
mode: parsedMode,
|
|
857
|
+
accountId: resolvedAccount.accountId,
|
|
858
|
+
pageSize: limit,
|
|
859
|
+
boundary: getMentionRequestBoundary({
|
|
860
|
+
sinceId: resolvedSinceId,
|
|
861
|
+
startTime: resolvedStartTime,
|
|
862
|
+
}),
|
|
863
|
+
};
|
|
864
|
+
const resultCacheKey = getMentionResultCacheKey({
|
|
865
|
+
shape: resultShape,
|
|
866
|
+
all: fetchAll,
|
|
867
|
+
maxPages: parsedMaxPages,
|
|
868
|
+
});
|
|
869
|
+
const ttlMs = parseCacheTtlMs(cacheTtlMs);
|
|
870
|
+
const cached = startPaginationToken
|
|
871
|
+
? null
|
|
872
|
+
: yield* trySync(() =>
|
|
873
|
+
readSyncCache<XurlMentionsResponse>(resultCacheKey, db),
|
|
874
|
+
);
|
|
875
|
+
const cachedPaginationToken = getCachedPaginationToken(cached);
|
|
876
|
+
const cacheAgeMs = cached
|
|
877
|
+
? Date.now() - new Date(cached.updatedAt).getTime()
|
|
878
|
+
: Number.POSITIVE_INFINITY;
|
|
872
879
|
|
|
873
|
-
|
|
874
|
-
|
|
875
|
-
|
|
876
|
-
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
|
|
885
|
-
|
|
886
|
-
db,
|
|
887
|
-
resolvedAccount.accountId,
|
|
888
|
-
payload,
|
|
889
|
-
parsedMode,
|
|
890
|
-
);
|
|
891
|
-
const payloadPaginationToken = getCachedPaginationToken({ value: payload });
|
|
892
|
-
if (parsedMode === "xurl") {
|
|
893
|
-
if (payloadPaginationToken) {
|
|
894
|
-
writeSyncCache(
|
|
895
|
-
cursorKey,
|
|
896
|
-
addMentionCursorState(
|
|
897
|
-
payload,
|
|
898
|
-
getMentionRequestBoundary({
|
|
899
|
-
sinceId: resolvedSinceId,
|
|
900
|
-
startTime: resolvedStartTime,
|
|
901
|
-
}),
|
|
902
|
-
maxNumericTweetId(resolvedSinceId, getNewestMentionId(payload)),
|
|
880
|
+
if (
|
|
881
|
+
!startPaginationToken &&
|
|
882
|
+
!cachedPaginationToken &&
|
|
883
|
+
!refresh &&
|
|
884
|
+
cached &&
|
|
885
|
+
cacheAgeMs <= ttlMs
|
|
886
|
+
) {
|
|
887
|
+
yield* trySync(() =>
|
|
888
|
+
mergeMentionsIntoLocalStore(
|
|
889
|
+
db,
|
|
890
|
+
resolvedAccount.accountId,
|
|
891
|
+
cached.value,
|
|
892
|
+
parsedMode,
|
|
903
893
|
),
|
|
904
|
-
db,
|
|
905
894
|
);
|
|
906
|
-
|
|
907
|
-
|
|
908
|
-
|
|
909
|
-
|
|
910
|
-
|
|
895
|
+
return {
|
|
896
|
+
ok: true,
|
|
897
|
+
source: "cache",
|
|
898
|
+
kind: "mentions",
|
|
899
|
+
accountId: resolvedAccount.accountId,
|
|
900
|
+
count: cached.value.data.length,
|
|
901
|
+
partial: isMaxPagesPartial({
|
|
902
|
+
payload: cached.value,
|
|
911
903
|
maxPages: parsedMaxPages,
|
|
912
904
|
}),
|
|
913
|
-
|
|
905
|
+
payload: cached.value,
|
|
906
|
+
};
|
|
907
|
+
}
|
|
908
|
+
|
|
909
|
+
if (
|
|
910
|
+
parsedMode === "xurl" &&
|
|
911
|
+
!explicitSinceId &&
|
|
912
|
+
!explicitStartTime &&
|
|
913
|
+
!startPaginationToken &&
|
|
914
|
+
!seededSinceId
|
|
915
|
+
) {
|
|
916
|
+
console.error(
|
|
917
|
+
"No local mention baseline found; syncing mentions from the newest page backwards.",
|
|
914
918
|
);
|
|
915
|
-
|
|
916
|
-
|
|
917
|
-
|
|
918
|
-
|
|
919
|
-
|
|
920
|
-
|
|
921
|
-
|
|
922
|
-
|
|
923
|
-
|
|
924
|
-
|
|
925
|
-
|
|
926
|
-
|
|
927
|
-
|
|
928
|
-
|
|
929
|
-
|
|
930
|
-
|
|
931
|
-
|
|
932
|
-
|
|
933
|
-
|
|
919
|
+
}
|
|
920
|
+
|
|
921
|
+
const payload =
|
|
922
|
+
parsedMode === "bird"
|
|
923
|
+
? yield* fetchMentionsViaBirdEffect({ limit })
|
|
924
|
+
: yield* fetchMentionsViaXurlEffect({
|
|
925
|
+
resolvedAccount,
|
|
926
|
+
limit,
|
|
927
|
+
all: fetchAll,
|
|
928
|
+
parsedMaxPages,
|
|
929
|
+
sinceId: resolvedSinceId,
|
|
930
|
+
startPaginationToken,
|
|
931
|
+
startTime: resolvedStartTime,
|
|
932
|
+
});
|
|
933
|
+
yield* trySync(() =>
|
|
934
|
+
mergeMentionsIntoLocalStore(
|
|
935
|
+
db,
|
|
936
|
+
resolvedAccount.accountId,
|
|
937
|
+
payload,
|
|
938
|
+
parsedMode,
|
|
939
|
+
),
|
|
940
|
+
);
|
|
941
|
+
const payloadPaginationToken = getCachedPaginationToken({ value: payload });
|
|
942
|
+
if (parsedMode === "xurl") {
|
|
943
|
+
if (payloadPaginationToken) {
|
|
944
|
+
yield* trySync(() => {
|
|
945
|
+
writeSyncCache(
|
|
946
|
+
cursorKey,
|
|
947
|
+
addMentionCursorState(
|
|
948
|
+
payload,
|
|
949
|
+
getMentionRequestBoundary({
|
|
950
|
+
sinceId: resolvedSinceId,
|
|
951
|
+
startTime: resolvedStartTime,
|
|
952
|
+
}),
|
|
953
|
+
maxNumericTweetId(resolvedSinceId, getNewestMentionId(payload)),
|
|
954
|
+
),
|
|
955
|
+
db,
|
|
956
|
+
);
|
|
957
|
+
deleteSyncCache(resultCacheKey, db);
|
|
958
|
+
deleteSyncCache(
|
|
959
|
+
getMentionResultCacheKey({
|
|
960
|
+
shape: cursorShape,
|
|
961
|
+
all: fetchAll,
|
|
962
|
+
maxPages: parsedMaxPages,
|
|
963
|
+
}),
|
|
964
|
+
db,
|
|
965
|
+
);
|
|
966
|
+
for (const legacyKey of cursor?.legacyKeys ?? []) {
|
|
967
|
+
deleteSyncCache(legacyKey, db);
|
|
968
|
+
}
|
|
969
|
+
});
|
|
970
|
+
} else {
|
|
971
|
+
yield* trySync(() => {
|
|
972
|
+
deleteSyncCache(cursorKey, db);
|
|
973
|
+
for (const legacyKey of legacyCursorKeys) {
|
|
974
|
+
deleteSyncCache(legacyKey, db);
|
|
975
|
+
}
|
|
976
|
+
if (cursorShape.boundary.kind === "auto") {
|
|
977
|
+
writeMentionHighWaterId(
|
|
978
|
+
db,
|
|
979
|
+
parsedMode,
|
|
980
|
+
resolvedAccount.accountId,
|
|
981
|
+
maxNumericTweetId(
|
|
982
|
+
resolvedSinceId,
|
|
983
|
+
cursor?.pendingNewestId,
|
|
984
|
+
getNewestMentionId(payload),
|
|
985
|
+
),
|
|
986
|
+
);
|
|
987
|
+
}
|
|
988
|
+
});
|
|
934
989
|
}
|
|
935
990
|
}
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
}
|
|
991
|
+
if (!payloadPaginationToken && !startPaginationToken) {
|
|
992
|
+
yield* trySync(() => writeSyncCache(resultCacheKey, payload, db));
|
|
993
|
+
}
|
|
940
994
|
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
|
|
946
|
-
|
|
947
|
-
|
|
948
|
-
|
|
949
|
-
|
|
995
|
+
return {
|
|
996
|
+
ok: true,
|
|
997
|
+
source: parsedMode,
|
|
998
|
+
kind: "mentions",
|
|
999
|
+
accountId: resolvedAccount.accountId,
|
|
1000
|
+
count: payload.data.length,
|
|
1001
|
+
partial: isMaxPagesPartial({ payload, maxPages: parsedMaxPages }),
|
|
1002
|
+
payload,
|
|
1003
|
+
};
|
|
1004
|
+
});
|
|
1005
|
+
}
|
|
1006
|
+
|
|
1007
|
+
export function syncMentions(options: SyncMentionsOptions) {
|
|
1008
|
+
return runEffectPromise(syncMentionsEffect(options));
|
|
950
1009
|
}
|
|
951
1010
|
|
|
952
|
-
|
|
1011
|
+
function exportMentionsViaCachedLiveSourceEffect({
|
|
953
1012
|
mode,
|
|
954
1013
|
account,
|
|
955
1014
|
search,
|
|
@@ -959,90 +1018,36 @@ async function exportMentionsViaCachedLiveSource({
|
|
|
959
1018
|
maxPages,
|
|
960
1019
|
refresh = false,
|
|
961
1020
|
cacheTtlMs,
|
|
962
|
-
}: {
|
|
963
|
-
|
|
964
|
-
|
|
965
|
-
|
|
966
|
-
|
|
967
|
-
|
|
968
|
-
all?: boolean;
|
|
969
|
-
maxPages?: number;
|
|
970
|
-
refresh?: boolean;
|
|
971
|
-
cacheTtlMs?: number;
|
|
972
|
-
}) {
|
|
973
|
-
if (mode === "xurl") {
|
|
974
|
-
assertXurlLimit(limit);
|
|
975
|
-
} else {
|
|
976
|
-
assertBirdLimit(limit);
|
|
977
|
-
}
|
|
978
|
-
const parsedMaxPages = parseMaxPages(maxPages);
|
|
979
|
-
const fetchAll = mode === "xurl" && (all || parsedMaxPages !== null);
|
|
980
|
-
|
|
981
|
-
const db = getNativeDb();
|
|
982
|
-
const resolvedAccount = resolveAccount(db, account);
|
|
983
|
-
const cacheKey = getMentionsFetchModeKey({
|
|
984
|
-
scope: "export",
|
|
985
|
-
mode,
|
|
986
|
-
accountId: resolvedAccount.accountId,
|
|
987
|
-
pageSize: limit,
|
|
988
|
-
all: fetchAll,
|
|
989
|
-
maxPages: parsedMaxPages,
|
|
990
|
-
sinceId: null,
|
|
991
|
-
startTime: null,
|
|
992
|
-
});
|
|
993
|
-
const ttlMs = parseCacheTtlMs(cacheTtlMs);
|
|
994
|
-
const cached = readSyncCache<XurlMentionsResponse>(cacheKey, db);
|
|
995
|
-
const cacheAgeMs = cached
|
|
996
|
-
? Date.now() - new Date(cached.updatedAt).getTime()
|
|
997
|
-
: Number.POSITIVE_INFINITY;
|
|
998
|
-
|
|
999
|
-
if (!refresh && cached && cacheAgeMs <= ttlMs) {
|
|
1000
|
-
if (
|
|
1001
|
-
shouldReturnFilteredLocalPayload({
|
|
1002
|
-
search,
|
|
1003
|
-
replyFilter,
|
|
1004
|
-
})
|
|
1005
|
-
) {
|
|
1006
|
-
return readLocalXurlCompatiblePayload({
|
|
1007
|
-
accountId: resolvedAccount.accountId,
|
|
1008
|
-
search,
|
|
1009
|
-
replyFilter,
|
|
1010
|
-
limit: fetchAll ? cached.value.data.length : limit,
|
|
1011
|
-
});
|
|
1012
|
-
}
|
|
1013
|
-
return cached.value;
|
|
1014
|
-
}
|
|
1015
|
-
|
|
1016
|
-
try {
|
|
1017
|
-
const payload =
|
|
1018
|
-
mode === "bird"
|
|
1019
|
-
? await listMentionsViaBird({ maxResults: limit })
|
|
1020
|
-
: await fetchMentionsViaXurl({
|
|
1021
|
-
resolvedAccount,
|
|
1022
|
-
limit,
|
|
1023
|
-
all: fetchAll,
|
|
1024
|
-
parsedMaxPages,
|
|
1025
|
-
});
|
|
1026
|
-
mergeMentionsIntoLocalStore(db, resolvedAccount.accountId, payload, mode);
|
|
1027
|
-
writeSyncCache(cacheKey, payload, db);
|
|
1028
|
-
|
|
1029
|
-
if (
|
|
1030
|
-
shouldReturnFilteredLocalPayload({
|
|
1031
|
-
search,
|
|
1032
|
-
replyFilter,
|
|
1033
|
-
})
|
|
1034
|
-
) {
|
|
1035
|
-
return readLocalXurlCompatiblePayload({
|
|
1036
|
-
accountId: resolvedAccount.accountId,
|
|
1037
|
-
search,
|
|
1038
|
-
replyFilter,
|
|
1039
|
-
limit: fetchAll ? payload.data.length : limit,
|
|
1040
|
-
});
|
|
1021
|
+
}: ExportMentionsViaCachedLiveSourceOptions) {
|
|
1022
|
+
return Effect.gen(function* () {
|
|
1023
|
+
if (mode === "xurl") {
|
|
1024
|
+
yield* trySync(() => assertXurlLimit(limit));
|
|
1025
|
+
} else {
|
|
1026
|
+
yield* trySync(() => assertBirdLimit(limit));
|
|
1041
1027
|
}
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
|
|
1045
|
-
|
|
1028
|
+
const parsedMaxPages = yield* trySync(() => parseMaxPages(maxPages));
|
|
1029
|
+
const fetchAll = mode === "xurl" && (all || parsedMaxPages !== null);
|
|
1030
|
+
|
|
1031
|
+
const db = yield* trySync(() => getNativeDb());
|
|
1032
|
+
const resolvedAccount = yield* trySync(() => resolveAccount(db, account));
|
|
1033
|
+
const cacheKey = getMentionsFetchModeKey({
|
|
1034
|
+
scope: "export",
|
|
1035
|
+
mode,
|
|
1036
|
+
accountId: resolvedAccount.accountId,
|
|
1037
|
+
pageSize: limit,
|
|
1038
|
+
all: fetchAll,
|
|
1039
|
+
maxPages: parsedMaxPages,
|
|
1040
|
+
sinceId: null,
|
|
1041
|
+
startTime: null,
|
|
1042
|
+
});
|
|
1043
|
+
const ttlMs = parseCacheTtlMs(cacheTtlMs);
|
|
1044
|
+
const cached = yield* trySync(() =>
|
|
1045
|
+
readSyncCache<XurlMentionsResponse>(cacheKey, db),
|
|
1046
|
+
);
|
|
1047
|
+
const cacheAgeMs = cached
|
|
1048
|
+
? Date.now() - new Date(cached.updatedAt).getTime()
|
|
1049
|
+
: Number.POSITIVE_INFINITY;
|
|
1050
|
+
const readFilteredOrRaw = (payload: XurlMentionsResponse) => {
|
|
1046
1051
|
if (
|
|
1047
1052
|
shouldReturnFilteredLocalPayload({
|
|
1048
1053
|
search,
|
|
@@ -1053,35 +1058,78 @@ async function exportMentionsViaCachedLiveSource({
|
|
|
1053
1058
|
accountId: resolvedAccount.accountId,
|
|
1054
1059
|
search,
|
|
1055
1060
|
replyFilter,
|
|
1056
|
-
limit: fetchAll ?
|
|
1061
|
+
limit: fetchAll ? payload.data.length : limit,
|
|
1057
1062
|
});
|
|
1058
1063
|
}
|
|
1059
|
-
return
|
|
1064
|
+
return payload;
|
|
1065
|
+
};
|
|
1066
|
+
|
|
1067
|
+
if (!refresh && cached && cacheAgeMs <= ttlMs) {
|
|
1068
|
+
return yield* trySync(() => readFilteredOrRaw(cached.value));
|
|
1060
1069
|
}
|
|
1061
|
-
|
|
1062
|
-
|
|
1070
|
+
|
|
1071
|
+
const liveResult = yield* (
|
|
1072
|
+
mode === "bird"
|
|
1073
|
+
? fetchMentionsViaBirdEffect({ limit })
|
|
1074
|
+
: fetchMentionsViaXurlEffect({
|
|
1075
|
+
resolvedAccount,
|
|
1076
|
+
limit,
|
|
1077
|
+
all: fetchAll,
|
|
1078
|
+
parsedMaxPages,
|
|
1079
|
+
})
|
|
1080
|
+
).pipe(
|
|
1081
|
+
Effect.flatMap((payload) =>
|
|
1082
|
+
trySync(() => {
|
|
1083
|
+
mergeMentionsIntoLocalStore(
|
|
1084
|
+
db,
|
|
1085
|
+
resolvedAccount.accountId,
|
|
1086
|
+
payload,
|
|
1087
|
+
mode,
|
|
1088
|
+
);
|
|
1089
|
+
writeSyncCache(cacheKey, payload, db);
|
|
1090
|
+
return readFilteredOrRaw(payload);
|
|
1091
|
+
}),
|
|
1092
|
+
),
|
|
1093
|
+
Effect.map((payload) => ({ ok: true as const, payload })),
|
|
1094
|
+
Effect.catchAll((error) => {
|
|
1095
|
+
if (!refresh && cached) {
|
|
1096
|
+
return Effect.succeed({ ok: false as const });
|
|
1097
|
+
}
|
|
1098
|
+
return Effect.fail(error);
|
|
1099
|
+
}),
|
|
1100
|
+
);
|
|
1101
|
+
|
|
1102
|
+
if (!liveResult.ok) {
|
|
1103
|
+
if (!cached) {
|
|
1104
|
+
return yield* Effect.fail(
|
|
1105
|
+
new Error("Mention export failed without cache"),
|
|
1106
|
+
);
|
|
1107
|
+
}
|
|
1108
|
+
return yield* trySync(() => readFilteredOrRaw(cached.value));
|
|
1109
|
+
}
|
|
1110
|
+
|
|
1111
|
+
return liveResult.payload;
|
|
1112
|
+
});
|
|
1063
1113
|
}
|
|
1064
1114
|
|
|
1065
|
-
export
|
|
1066
|
-
options: Omit<
|
|
1067
|
-
Parameters<typeof exportMentionsViaCachedLiveSource>[0],
|
|
1068
|
-
"mode"
|
|
1069
|
-
>,
|
|
1115
|
+
export function exportMentionsViaCachedXurl(
|
|
1116
|
+
options: Omit<ExportMentionsViaCachedLiveSourceOptions, "mode">,
|
|
1070
1117
|
) {
|
|
1071
|
-
return
|
|
1072
|
-
|
|
1073
|
-
|
|
1074
|
-
|
|
1118
|
+
return runEffectPromise(
|
|
1119
|
+
exportMentionsViaCachedLiveSourceEffect({
|
|
1120
|
+
...options,
|
|
1121
|
+
mode: "xurl",
|
|
1122
|
+
}),
|
|
1123
|
+
);
|
|
1075
1124
|
}
|
|
1076
1125
|
|
|
1077
|
-
export
|
|
1078
|
-
options: Omit<
|
|
1079
|
-
Parameters<typeof exportMentionsViaCachedLiveSource>[0],
|
|
1080
|
-
"mode"
|
|
1081
|
-
>,
|
|
1126
|
+
export function exportMentionsViaCachedBird(
|
|
1127
|
+
options: Omit<ExportMentionsViaCachedLiveSourceOptions, "mode">,
|
|
1082
1128
|
) {
|
|
1083
|
-
return
|
|
1084
|
-
|
|
1085
|
-
|
|
1086
|
-
|
|
1129
|
+
return runEffectPromise(
|
|
1130
|
+
exportMentionsViaCachedLiveSourceEffect({
|
|
1131
|
+
...options,
|
|
1132
|
+
mode: "bird",
|
|
1133
|
+
}),
|
|
1134
|
+
);
|
|
1087
1135
|
}
|