@atproto/bsky 0.0.16 → 0.0.17
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +7 -0
- package/dist/cache/read-through.d.ts +30 -0
- package/dist/config.d.ts +18 -0
- package/dist/context.d.ts +6 -6
- package/dist/daemon/config.d.ts +15 -0
- package/dist/daemon/context.d.ts +15 -0
- package/dist/daemon/index.d.ts +23 -0
- package/dist/daemon/logger.d.ts +3 -0
- package/dist/daemon/notifications.d.ts +18 -0
- package/dist/daemon/services.d.ts +11 -0
- package/dist/db/database-schema.d.ts +1 -2
- package/dist/db/index.js +16 -1
- package/dist/db/index.js.map +3 -3
- package/dist/db/migrations/20231205T000257238Z-remove-did-cache.d.ts +3 -0
- package/dist/db/migrations/index.d.ts +1 -0
- package/dist/did-cache.d.ts +10 -7
- package/dist/index.d.ts +4 -0
- package/dist/index.js +1917 -934
- package/dist/index.js.map +3 -3
- package/dist/indexer/context.d.ts +2 -0
- package/dist/indexer/index.d.ts +1 -0
- package/dist/lexicon/index.d.ts +12 -0
- package/dist/lexicon/lexicons.d.ts +134 -0
- package/dist/lexicon/types/com/atproto/admin/deleteAccount.d.ts +25 -0
- package/dist/lexicon/types/com/atproto/temp/importRepo.d.ts +32 -0
- package/dist/lexicon/types/com/atproto/temp/pushBlob.d.ts +25 -0
- package/dist/lexicon/types/com/atproto/temp/transferAccount.d.ts +42 -0
- package/dist/logger.d.ts +1 -0
- package/dist/redis.d.ts +10 -1
- package/dist/services/actor/index.d.ts +18 -4
- package/dist/services/actor/views.d.ts +4 -3
- package/dist/services/feed/index.d.ts +6 -4
- package/dist/services/feed/views.d.ts +5 -4
- package/dist/services/index.d.ts +3 -7
- package/dist/services/label/index.d.ts +10 -4
- package/dist/services/moderation/index.d.ts +0 -1
- package/dist/services/types.d.ts +3 -0
- package/dist/services/util/notification.d.ts +5 -0
- package/dist/services/util/post.d.ts +6 -6
- package/dist/util/retry.d.ts +1 -6
- package/package.json +5 -5
- package/src/cache/read-through.ts +151 -0
- package/src/config.ts +90 -1
- package/src/context.ts +7 -7
- package/src/daemon/config.ts +60 -0
- package/src/daemon/context.ts +27 -0
- package/src/daemon/index.ts +78 -0
- package/src/daemon/logger.ts +6 -0
- package/src/daemon/notifications.ts +54 -0
- package/src/daemon/services.ts +22 -0
- package/src/db/database-schema.ts +0 -2
- package/src/db/migrations/20231205T000257238Z-remove-did-cache.ts +14 -0
- package/src/db/migrations/index.ts +1 -0
- package/src/did-cache.ts +33 -56
- package/src/feed-gen/index.ts +0 -4
- package/src/index.ts +55 -16
- package/src/indexer/context.ts +5 -0
- package/src/indexer/index.ts +10 -7
- package/src/lexicon/index.ts +50 -0
- package/src/lexicon/lexicons.ts +156 -0
- package/src/lexicon/types/com/atproto/admin/deleteAccount.ts +38 -0
- package/src/lexicon/types/com/atproto/temp/importRepo.ts +45 -0
- package/src/lexicon/types/com/atproto/temp/pushBlob.ts +39 -0
- package/src/lexicon/types/com/atproto/temp/transferAccount.ts +62 -0
- package/src/logger.ts +2 -0
- package/src/redis.ts +43 -3
- package/src/services/actor/index.ts +55 -7
- package/src/services/actor/views.ts +13 -7
- package/src/services/feed/index.ts +27 -13
- package/src/services/feed/views.ts +20 -10
- package/src/services/index.ts +14 -14
- package/src/services/indexing/index.ts +7 -10
- package/src/services/indexing/plugins/post.ts +13 -0
- package/src/services/label/index.ts +66 -22
- package/src/services/moderation/index.ts +1 -1
- package/src/services/moderation/status.ts +1 -4
- package/src/services/types.ts +4 -0
- package/src/services/util/notification.ts +70 -0
- package/src/util/retry.ts +1 -44
- package/tests/admin/get-repo.test.ts +5 -3
- package/tests/admin/moderation.test.ts +2 -2
- package/tests/admin/repo-search.test.ts +1 -0
- package/tests/algos/hot-classic.test.ts +1 -2
- package/tests/auth.test.ts +1 -1
- package/tests/auto-moderator/labeler.test.ts +19 -20
- package/tests/auto-moderator/takedowns.test.ts +16 -10
- package/tests/blob-resolver.test.ts +4 -2
- package/tests/daemon.test.ts +191 -0
- package/tests/did-cache.test.ts +20 -5
- package/tests/handle-invalidation.test.ts +1 -5
- package/tests/indexing.test.ts +20 -13
- package/tests/redis-cache.test.ts +231 -0
- package/tests/seeds/basic.ts +3 -0
- package/tests/subscription/repo.test.ts +4 -7
- package/tests/views/profile.test.ts +0 -1
- package/tests/views/thread.test.ts +73 -78
- package/tests/views/threadgating.test.ts +38 -0
- package/dist/db/tables/did-cache.d.ts +0 -10
- package/dist/feed-gen/best-of-follows.d.ts +0 -29
- package/dist/feed-gen/whats-hot.d.ts +0 -29
- package/dist/feed-gen/with-friends.d.ts +0 -3
- package/dist/label-cache.d.ts +0 -19
- package/src/db/tables/did-cache.ts +0 -13
- package/src/feed-gen/best-of-follows.ts +0 -77
- package/src/feed-gen/whats-hot.ts +0 -101
- package/src/feed-gen/with-friends.ts +0 -43
- package/src/label-cache.ts +0 -90
- package/tests/algos/whats-hot.test.ts +0 -118
- package/tests/algos/with-friends.test.ts +0 -145
package/CHANGELOG.md
CHANGED
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
import { Redis } from '../redis';
|
|
2
|
+
export declare type CacheItem<T> = {
|
|
3
|
+
val: T | null;
|
|
4
|
+
updatedAt: number;
|
|
5
|
+
};
|
|
6
|
+
export declare type CacheOptions<T> = {
|
|
7
|
+
staleTTL: number;
|
|
8
|
+
maxTTL: number;
|
|
9
|
+
fetchMethod: (key: string) => Promise<T | null>;
|
|
10
|
+
fetchManyMethod?: (keys: string[]) => Promise<Record<string, T | null>>;
|
|
11
|
+
};
|
|
12
|
+
export declare class ReadThroughCache<T> {
|
|
13
|
+
redis: Redis;
|
|
14
|
+
opts: CacheOptions<T>;
|
|
15
|
+
constructor(redis: Redis, opts: CacheOptions<T>);
|
|
16
|
+
private _fetchMany;
|
|
17
|
+
private fetchAndCache;
|
|
18
|
+
private fetchAndCacheMany;
|
|
19
|
+
get(key: string, opts?: {
|
|
20
|
+
revalidate?: boolean;
|
|
21
|
+
}): Promise<T | null>;
|
|
22
|
+
getMany(keys: string[], opts?: {
|
|
23
|
+
revalidate?: boolean;
|
|
24
|
+
}): Promise<Record<string, T>>;
|
|
25
|
+
set(key: string, val: T | null): Promise<void>;
|
|
26
|
+
setMany(vals: Record<string, T | null>): Promise<void>;
|
|
27
|
+
clearEntry(key: string): Promise<void>;
|
|
28
|
+
isExpired(result: CacheItem<T>): boolean;
|
|
29
|
+
isStale(result: CacheItem<T>): boolean;
|
|
30
|
+
}
|
package/dist/config.d.ts
CHANGED
|
@@ -9,9 +9,15 @@ export interface ServerConfigValues {
|
|
|
9
9
|
dbReplicaPostgresUrls?: string[];
|
|
10
10
|
dbReplicaTags?: Record<string, number[]>;
|
|
11
11
|
dbPostgresSchema?: string;
|
|
12
|
+
redisHost?: string;
|
|
13
|
+
redisSentinelName?: string;
|
|
14
|
+
redisSentinelHosts?: string[];
|
|
15
|
+
redisPassword?: string;
|
|
12
16
|
didPlcUrl: string;
|
|
13
17
|
didCacheStaleTTL: number;
|
|
14
18
|
didCacheMaxTTL: number;
|
|
19
|
+
labelCacheStaleTTL: number;
|
|
20
|
+
labelCacheMaxTTL: number;
|
|
15
21
|
handleResolveNameservers?: string[];
|
|
16
22
|
imgUriEndpoint?: string;
|
|
17
23
|
blobCacheLocation?: string;
|
|
@@ -21,6 +27,9 @@ export interface ServerConfigValues {
|
|
|
21
27
|
moderatorPassword?: string;
|
|
22
28
|
triagePassword?: string;
|
|
23
29
|
moderationPushUrl?: string;
|
|
30
|
+
rateLimitsEnabled: boolean;
|
|
31
|
+
rateLimitBypassKey?: string;
|
|
32
|
+
rateLimitBypassIps?: string[];
|
|
24
33
|
}
|
|
25
34
|
export declare class ServerConfig {
|
|
26
35
|
private cfg;
|
|
@@ -39,8 +48,14 @@ export declare class ServerConfig {
|
|
|
39
48
|
get dbReplicaPostgresUrl(): string[] | undefined;
|
|
40
49
|
get dbReplicaTags(): Record<string, number[]> | undefined;
|
|
41
50
|
get dbPostgresSchema(): string | undefined;
|
|
51
|
+
get redisHost(): string | undefined;
|
|
52
|
+
get redisSentinelName(): string | undefined;
|
|
53
|
+
get redisSentinelHosts(): string[] | undefined;
|
|
54
|
+
get redisPassword(): string | undefined;
|
|
42
55
|
get didCacheStaleTTL(): number;
|
|
43
56
|
get didCacheMaxTTL(): number;
|
|
57
|
+
get labelCacheStaleTTL(): number;
|
|
58
|
+
get labelCacheMaxTTL(): number;
|
|
44
59
|
get handleResolveNameservers(): string[] | undefined;
|
|
45
60
|
get didPlcUrl(): string;
|
|
46
61
|
get imgUriEndpoint(): string | undefined;
|
|
@@ -51,4 +66,7 @@ export declare class ServerConfig {
|
|
|
51
66
|
get moderatorPassword(): string | undefined;
|
|
52
67
|
get triagePassword(): string | undefined;
|
|
53
68
|
get moderationPushUrl(): string | undefined;
|
|
69
|
+
get rateLimitsEnabled(): boolean;
|
|
70
|
+
get rateLimitBypassKey(): string | undefined;
|
|
71
|
+
get rateLimitBypassIps(): string[] | undefined;
|
|
54
72
|
}
|
package/dist/context.d.ts
CHANGED
|
@@ -8,11 +8,11 @@ import { DatabaseCoordinator } from './db';
|
|
|
8
8
|
import { ServerConfig } from './config';
|
|
9
9
|
import { ImageUriBuilder } from './image/uri';
|
|
10
10
|
import { Services } from './services';
|
|
11
|
-
import
|
|
11
|
+
import DidRedisCache from './did-cache';
|
|
12
12
|
import { BackgroundQueue } from './background';
|
|
13
13
|
import { MountedAlgos } from './feed-gen/types';
|
|
14
|
-
import { LabelCache } from './label-cache';
|
|
15
14
|
import { NotificationServer } from './notifications';
|
|
15
|
+
import { Redis } from './redis';
|
|
16
16
|
export declare class AppContext {
|
|
17
17
|
private opts;
|
|
18
18
|
moderationPushAgent: AtpAgent | undefined;
|
|
@@ -23,8 +23,8 @@ export declare class AppContext {
|
|
|
23
23
|
services: Services;
|
|
24
24
|
signingKey: Keypair;
|
|
25
25
|
idResolver: IdResolver;
|
|
26
|
-
didCache:
|
|
27
|
-
|
|
26
|
+
didCache: DidRedisCache;
|
|
27
|
+
redis: Redis;
|
|
28
28
|
backgroundQueue: BackgroundQueue;
|
|
29
29
|
searchAgent?: AtpAgent;
|
|
30
30
|
algos: MountedAlgos;
|
|
@@ -37,8 +37,8 @@ export declare class AppContext {
|
|
|
37
37
|
get signingKey(): Keypair;
|
|
38
38
|
get plcClient(): plc.Client;
|
|
39
39
|
get idResolver(): IdResolver;
|
|
40
|
-
get didCache():
|
|
41
|
-
get
|
|
40
|
+
get didCache(): DidRedisCache;
|
|
41
|
+
get redis(): Redis;
|
|
42
42
|
get notifServer(): NotificationServer;
|
|
43
43
|
get searchAgent(): AtpAgent | undefined;
|
|
44
44
|
get authVerifier(): (reqCtx: {
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export interface DaemonConfigValues {
|
|
2
|
+
version: string;
|
|
3
|
+
dbPostgresUrl: string;
|
|
4
|
+
dbPostgresSchema?: string;
|
|
5
|
+
notificationsDaemonFromDid?: string;
|
|
6
|
+
}
|
|
7
|
+
export declare class DaemonConfig {
|
|
8
|
+
private cfg;
|
|
9
|
+
constructor(cfg: DaemonConfigValues);
|
|
10
|
+
static readEnv(overrides?: Partial<DaemonConfigValues>): DaemonConfig;
|
|
11
|
+
get version(): string;
|
|
12
|
+
get dbPostgresUrl(): string;
|
|
13
|
+
get dbPostgresSchema(): string | undefined;
|
|
14
|
+
get notificationsDaemonFromDid(): string | undefined;
|
|
15
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { PrimaryDatabase } from '../db';
|
|
2
|
+
import { DaemonConfig } from './config';
|
|
3
|
+
import { Services } from './services';
|
|
4
|
+
export declare class DaemonContext {
|
|
5
|
+
private opts;
|
|
6
|
+
constructor(opts: {
|
|
7
|
+
db: PrimaryDatabase;
|
|
8
|
+
cfg: DaemonConfig;
|
|
9
|
+
services: Services;
|
|
10
|
+
});
|
|
11
|
+
get db(): PrimaryDatabase;
|
|
12
|
+
get cfg(): DaemonConfig;
|
|
13
|
+
get services(): Services;
|
|
14
|
+
}
|
|
15
|
+
export default DaemonContext;
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { PrimaryDatabase } from '../db';
|
|
2
|
+
import { DaemonConfig } from './config';
|
|
3
|
+
import { DaemonContext } from './context';
|
|
4
|
+
import { NotificationsDaemon } from './notifications';
|
|
5
|
+
export { DaemonConfig } from './config';
|
|
6
|
+
export type { DaemonConfigValues } from './config';
|
|
7
|
+
export declare class BskyDaemon {
|
|
8
|
+
ctx: DaemonContext;
|
|
9
|
+
notifications: NotificationsDaemon;
|
|
10
|
+
private dbStatsInterval;
|
|
11
|
+
private notifStatsInterval;
|
|
12
|
+
constructor(opts: {
|
|
13
|
+
ctx: DaemonContext;
|
|
14
|
+
notifications: NotificationsDaemon;
|
|
15
|
+
});
|
|
16
|
+
static create(opts: {
|
|
17
|
+
db: PrimaryDatabase;
|
|
18
|
+
cfg: DaemonConfig;
|
|
19
|
+
}): BskyDaemon;
|
|
20
|
+
start(): Promise<this>;
|
|
21
|
+
destroy(): Promise<void>;
|
|
22
|
+
}
|
|
23
|
+
export default BskyDaemon;
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
import DaemonContext from './context';
|
|
2
|
+
export declare class NotificationsDaemon {
|
|
3
|
+
private ctx;
|
|
4
|
+
ac: AbortController;
|
|
5
|
+
running: Promise<void> | undefined;
|
|
6
|
+
count: number;
|
|
7
|
+
lastDid: string | null;
|
|
8
|
+
constructor(ctx: DaemonContext);
|
|
9
|
+
run(opts?: RunOptions): void;
|
|
10
|
+
private tidyNotifications;
|
|
11
|
+
destroy(): Promise<void>;
|
|
12
|
+
}
|
|
13
|
+
declare type RunOptions = {
|
|
14
|
+
forever?: boolean;
|
|
15
|
+
batchSize?: number;
|
|
16
|
+
startFromDid?: string;
|
|
17
|
+
};
|
|
18
|
+
export {};
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { PrimaryDatabase } from '../db';
|
|
2
|
+
import { ActorService } from '../services/actor';
|
|
3
|
+
import { ImageUriBuilder } from '../image/uri';
|
|
4
|
+
export declare function createServices(resources: {
|
|
5
|
+
imgUriBuilder: ImageUriBuilder;
|
|
6
|
+
}): Services;
|
|
7
|
+
export declare type Services = {
|
|
8
|
+
actor: FromDbPrimary<ActorService>;
|
|
9
|
+
};
|
|
10
|
+
declare type FromDbPrimary<T> = (db: PrimaryDatabase) => T;
|
|
11
|
+
export {};
|
|
@@ -24,13 +24,12 @@ import * as actorSync from './tables/actor-sync';
|
|
|
24
24
|
import * as record from './tables/record';
|
|
25
25
|
import * as notification from './tables/notification';
|
|
26
26
|
import * as notificationPushToken from './tables/notification-push-token';
|
|
27
|
-
import * as didCache from './tables/did-cache';
|
|
28
27
|
import * as moderation from './tables/moderation';
|
|
29
28
|
import * as label from './tables/label';
|
|
30
29
|
import * as algo from './tables/algo';
|
|
31
30
|
import * as viewParam from './tables/view-param';
|
|
32
31
|
import * as suggestedFollow from './tables/suggested-follow';
|
|
33
32
|
import * as suggestedFeed from './tables/suggested-feed';
|
|
34
|
-
export declare type DatabaseSchemaType = duplicateRecord.PartialDB & profile.PartialDB & profileAgg.PartialDB & post.PartialDB & postEmbed.PartialDB & postAgg.PartialDB & repost.PartialDB & threadGate.PartialDB & feedItem.PartialDB & follow.PartialDB & like.PartialDB & list.PartialDB & listItem.PartialDB & listMute.PartialDB & listBlock.PartialDB & mute.PartialDB & actorBlock.PartialDB & feedGenerator.PartialDB & subscription.PartialDB & actor.PartialDB & actorState.PartialDB & actorSync.PartialDB & record.PartialDB & notification.PartialDB & notificationPushToken.PartialDB &
|
|
33
|
+
export declare type DatabaseSchemaType = duplicateRecord.PartialDB & profile.PartialDB & profileAgg.PartialDB & post.PartialDB & postEmbed.PartialDB & postAgg.PartialDB & repost.PartialDB & threadGate.PartialDB & feedItem.PartialDB & follow.PartialDB & like.PartialDB & list.PartialDB & listItem.PartialDB & listMute.PartialDB & listBlock.PartialDB & mute.PartialDB & actorBlock.PartialDB & feedGenerator.PartialDB & subscription.PartialDB & actor.PartialDB & actorState.PartialDB & actorSync.PartialDB & record.PartialDB & notification.PartialDB & notificationPushToken.PartialDB & moderation.PartialDB & label.PartialDB & algo.PartialDB & viewParam.PartialDB & suggestedFollow.PartialDB & suggestedFeed.PartialDB;
|
|
35
34
|
export declare type DatabaseSchema = Kysely<DatabaseSchemaType>;
|
|
36
35
|
export default DatabaseSchema;
|
package/dist/db/index.js
CHANGED
|
@@ -33503,7 +33503,8 @@ __export(migrations_exports, {
|
|
|
33503
33503
|
_20230906T222220386Z: () => T222220386Z_thread_gating_exports,
|
|
33504
33504
|
_20230920T213858047Z: () => T213858047Z_add_tags_to_post_exports,
|
|
33505
33505
|
_20230929T192920807Z: () => T192920807Z_record_cursor_indexes_exports,
|
|
33506
|
-
_20231003T202833377Z: () => T202833377Z_create_moderation_subject_status_exports
|
|
33506
|
+
_20231003T202833377Z: () => T202833377Z_create_moderation_subject_status_exports,
|
|
33507
|
+
_20231205T000257238Z: () => T000257238Z_remove_did_cache_exports
|
|
33507
33508
|
});
|
|
33508
33509
|
|
|
33509
33510
|
// src/db/migrations/20230309T045948368Z-init.ts
|
|
@@ -34026,6 +34027,19 @@ async function down29(db) {
|
|
|
34026
34027
|
await db.schema.alterTable("record").addForeignKeyConstraint("record_takedown_id_fkey", ["takedownId"], "moderation_action", ["id"]).execute();
|
|
34027
34028
|
}
|
|
34028
34029
|
|
|
34030
|
+
// src/db/migrations/20231205T000257238Z-remove-did-cache.ts
|
|
34031
|
+
var T000257238Z_remove_did_cache_exports = {};
|
|
34032
|
+
__export(T000257238Z_remove_did_cache_exports, {
|
|
34033
|
+
down: () => down30,
|
|
34034
|
+
up: () => up30
|
|
34035
|
+
});
|
|
34036
|
+
async function up30(db) {
|
|
34037
|
+
await db.schema.dropTable("did_cache").execute();
|
|
34038
|
+
}
|
|
34039
|
+
async function down30(db) {
|
|
34040
|
+
await db.schema.createTable("did_cache").addColumn("did", "varchar", (col) => col.primaryKey()).addColumn("doc", "jsonb", (col) => col.notNull()).addColumn("updatedAt", "bigint", (col) => col.notNull()).execute();
|
|
34041
|
+
}
|
|
34042
|
+
|
|
34029
34043
|
// src/db/migrations/provider.ts
|
|
34030
34044
|
var CtxMigrationProvider = class {
|
|
34031
34045
|
constructor(migrations, ctx) {
|
|
@@ -34047,6 +34061,7 @@ var CtxMigrationProvider = class {
|
|
|
34047
34061
|
// src/logger.ts
|
|
34048
34062
|
var import_pino_http = __toESM(require_logger());
|
|
34049
34063
|
var dbLogger = subsystemLogger("bsky:db");
|
|
34064
|
+
var cacheLogger = subsystemLogger("bsky:cache");
|
|
34050
34065
|
var subLogger = subsystemLogger("bsky:sub");
|
|
34051
34066
|
var labelerLogger = subsystemLogger("bsky:labeler");
|
|
34052
34067
|
var httpLogger = subsystemLogger("bsky");
|