@clioplaylists/clio 0.1.7 → 0.1.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/api/com/clioplaylists/alpha/feed/getRecommendedPlaylistsByUser.d.ts +3 -0
- package/dist/api/com/clioplaylists/alpha/feed/getRecommendedPlaylistsByUser.js +70 -0
- package/dist/api/health.js +10 -3
- package/dist/api/index.d.ts +1 -0
- package/dist/api/index.js +40 -2
- package/dist/api/oauth.d.ts +3 -0
- package/dist/api/oauth.js +63 -0
- package/dist/api/util.d.ts +1 -0
- package/dist/api/util.js +11 -6
- package/dist/api/well-known.d.ts +3 -0
- package/dist/api/well-known.js +35 -0
- package/dist/auth-verifier.d.ts +17 -4
- package/dist/auth-verifier.js +171 -165
- package/dist/client.js +15 -8
- package/dist/config.d.ts +23 -0
- package/dist/config.js +87 -7
- package/dist/context.d.ts +14 -0
- package/dist/context.js +17 -1
- package/dist/dataplane/client/hosts.d.ts +21 -0
- package/dist/dataplane/client/hosts.js +29 -0
- package/dist/dataplane/client/index.d.ts +13 -0
- package/dist/dataplane/client/index.js +120 -0
- package/dist/dataplane/client/util.d.ts +20 -0
- package/dist/dataplane/client/util.js +92 -0
- package/dist/dataplane/index.js +18 -2
- package/dist/dataplane/server/background.d.ts +1 -1
- package/dist/dataplane/server/background.js +12 -5
- package/dist/dataplane/server/db/database-schema.d.ts +5 -1
- package/dist/dataplane/server/db/database-schema.js +2 -1
- package/dist/dataplane/server/db/db.js +60 -20
- package/dist/dataplane/server/db/index.js +17 -1
- package/dist/dataplane/server/db/migrations/20250515T045948368Z-init.d.ts +3 -0
- package/dist/dataplane/server/db/migrations/20250515T045948368Z-init.js +170 -0
- package/dist/dataplane/server/db/migrations/20260119T210000000Z-song-recommendation.d.ts +3 -0
- package/dist/dataplane/server/db/migrations/20260119T210000000Z-song-recommendation.js +36 -0
- package/dist/dataplane/server/db/migrations/20260119T220000000Z-oauth.d.ts +3 -0
- package/dist/dataplane/server/db/migrations/20260119T220000000Z-oauth.js +25 -0
- package/dist/dataplane/server/db/migrations/index.d.ts +3 -2
- package/dist/dataplane/server/db/migrations/index.js +39 -2
- package/dist/dataplane/server/db/migrations/provider.js +5 -1
- package/dist/dataplane/server/db/pagination.d.ts +1 -1
- package/dist/dataplane/server/db/pagination.js +38 -25
- package/dist/dataplane/server/db/tables/actor-sync.d.ts +2 -2
- package/dist/dataplane/server/db/tables/actor-sync.js +4 -1
- package/dist/dataplane/server/db/tables/actor.d.ts +3 -3
- package/dist/dataplane/server/db/tables/actor.js +4 -1
- package/dist/dataplane/server/db/tables/artist-list-item.d.ts +2 -2
- package/dist/dataplane/server/db/tables/artist-list-item.js +4 -1
- package/dist/dataplane/server/db/tables/artist.js +4 -1
- package/dist/dataplane/server/db/tables/oauth-session.d.ts +9 -0
- package/dist/dataplane/server/db/tables/oauth-session.js +4 -0
- package/dist/dataplane/server/db/tables/oauth-state.d.ts +10 -0
- package/dist/dataplane/server/db/tables/oauth-state.js +4 -0
- package/dist/dataplane/server/db/tables/playlist-idea.d.ts +3 -3
- package/dist/dataplane/server/db/tables/playlist-idea.js +4 -1
- package/dist/dataplane/server/db/tables/playlist-item.js +4 -1
- package/dist/dataplane/server/db/tables/playlist.d.ts +1 -0
- package/dist/dataplane/server/db/tables/playlist.js +4 -1
- package/dist/dataplane/server/db/tables/profile.d.ts +5 -5
- package/dist/dataplane/server/db/tables/profile.js +4 -1
- package/dist/dataplane/server/db/tables/record.d.ts +1 -1
- package/dist/dataplane/server/db/tables/record.js +4 -1
- package/dist/dataplane/server/db/tables/song-recommendation.d.ts +17 -0
- package/dist/dataplane/server/db/tables/song-recommendation.js +4 -0
- package/dist/dataplane/server/db/tables/song.d.ts +2 -2
- package/dist/dataplane/server/db/tables/song.js +4 -1
- package/dist/dataplane/server/db/tables/subscription-cursor.d.ts +9 -0
- package/dist/dataplane/server/db/tables/subscription-cursor.js +4 -0
- package/dist/dataplane/server/db/types.js +2 -1
- package/dist/dataplane/server/db/util.d.ts +7 -3
- package/dist/dataplane/server/db/util.js +26 -18
- package/dist/dataplane/server/index.js +21 -13
- package/dist/dataplane/server/indexing/index.d.ts +2 -0
- package/dist/dataplane/server/indexing/index.js +82 -66
- package/dist/dataplane/server/indexing/plugins/playlist-idea.d.ts +1 -2
- package/dist/dataplane/server/indexing/plugins/playlist-idea.js +50 -41
- package/dist/dataplane/server/indexing/plugins/profile.js +45 -12
- package/dist/dataplane/server/indexing/plugins/song-recommendation.d.ts +9 -0
- package/dist/dataplane/server/indexing/plugins/song-recommendation.js +101 -0
- package/dist/dataplane/server/indexing/processor.js +12 -11
- package/dist/dataplane/server/routes/identity.d.ts +19 -0
- package/dist/dataplane/server/routes/identity.js +32 -25
- package/dist/dataplane/server/routes/index.js +15 -10
- package/dist/dataplane/server/routes/profile.js +17 -25
- package/dist/dataplane/server/routes/records.d.ts +18 -0
- package/dist/dataplane/server/routes/records.js +48 -22
- package/dist/dataplane/server/routes/sync.js +5 -3
- package/dist/dataplane/server/storage/subscription-cursor.d.ts +3 -0
- package/dist/dataplane/server/storage/subscription-cursor.js +25 -0
- package/dist/dataplane/server/subscription.d.ts +6 -3
- package/dist/dataplane/server/subscription.js +73 -63
- package/dist/error.js +9 -5
- package/dist/index.d.ts +8 -3
- package/dist/index.js +96 -27
- package/dist/lexicons/index.d.ts +3 -210
- package/dist/lexicons/index.js +26 -403
- package/dist/lexicons/lexicons.d.ts +409 -8107
- package/dist/lexicons/lexicons.js +134 -4276
- package/dist/lexicons/types/com/atproto/repo/strongRef.d.ts +4 -4
- package/dist/lexicons/types/com/atproto/repo/strongRef.js +13 -9
- package/dist/lexicons/types/com/clioplaylists/alpha/actor/profile.d.ts +10 -8
- package/dist/lexicons/types/com/clioplaylists/alpha/actor/profile.js +13 -9
- package/dist/lexicons/types/com/clioplaylists/alpha/feed/defs.d.ts +20 -11
- package/dist/lexicons/types/com/clioplaylists/alpha/feed/defs.js +29 -14
- package/dist/lexicons/types/com/clioplaylists/alpha/feed/getRecommendedPlaylistsByUser.d.ts +36 -0
- package/dist/lexicons/types/com/clioplaylists/alpha/feed/getRecommendedPlaylistsByUser.js +6 -0
- package/dist/lexicons/types/com/clioplaylists/alpha/feed/playlistIdea.d.ts +11 -14
- package/dist/lexicons/types/com/clioplaylists/alpha/feed/playlistIdea.js +20 -23
- package/dist/lexicons/types/com/clioplaylists/alpha/feed/recommendedPlaylist.d.ts +18 -0
- package/dist/lexicons/types/com/clioplaylists/alpha/feed/recommendedPlaylist.js +15 -0
- package/dist/lexicons/types/com/clioplaylists/alpha/feed/songRecommendation.d.ts +18 -0
- package/dist/lexicons/types/com/clioplaylists/alpha/feed/songRecommendation.js +15 -0
- package/dist/lexicons/util.d.ts +33 -2
- package/dist/lexicons/util.js +32 -4
- package/dist/logger.js +16 -10
- package/dist/oauth/client.d.ts +14 -0
- package/dist/oauth/client.js +126 -0
- package/dist/oauth/pds-agent.d.ts +3 -0
- package/dist/oauth/pds-agent.js +15 -0
- package/dist/rpc/clio_connect.d.ts +101 -11
- package/dist/rpc/clio_connect.js +138 -45
- package/dist/rpc/clio_pb.d.ts +448 -30
- package/dist/rpc/clio_pb.js +967 -272
- package/dist/start.js +9 -1
- package/dist/util/retry.js +10 -9
- package/dist/util/uris.js +6 -3
- package/dist/util.d.ts +0 -1
- package/dist/util.js +61 -20
- package/package.json +26 -5
|
@@ -1,12 +1,12 @@
|
|
|
1
|
+
import { Generated } from 'kysely';
|
|
1
2
|
export declare const tableName = "playlist_idea";
|
|
2
3
|
export interface PlaylistIdeaTable {
|
|
4
|
+
id: Generated<number>;
|
|
3
5
|
uri: string;
|
|
4
6
|
title: string;
|
|
5
|
-
description
|
|
7
|
+
description: string | null;
|
|
6
8
|
creator_did: string;
|
|
7
9
|
owner_did: string;
|
|
8
|
-
master_playlist_id?: number;
|
|
9
|
-
recommended_playlist_id?: number;
|
|
10
10
|
created_at: string;
|
|
11
11
|
}
|
|
12
12
|
export type PartialDB = {
|
|
@@ -2,11 +2,11 @@ export declare const tableName = "profile";
|
|
|
2
2
|
export interface ProfileTable {
|
|
3
3
|
uri: string;
|
|
4
4
|
cid: string;
|
|
5
|
-
|
|
6
|
-
display_name
|
|
7
|
-
description
|
|
8
|
-
avatar_cid
|
|
9
|
-
banner_cid
|
|
5
|
+
creator_did: string;
|
|
6
|
+
display_name: string | null;
|
|
7
|
+
description: string | null;
|
|
8
|
+
avatar_cid: string | null;
|
|
9
|
+
banner_cid: string | null;
|
|
10
10
|
created_at: string;
|
|
11
11
|
indexed_at: string;
|
|
12
12
|
}
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { Generated } from 'kysely';
|
|
2
|
+
export declare const tableName = "song_recommendation";
|
|
3
|
+
export interface SongRecommendationTable {
|
|
4
|
+
id: Generated<number>;
|
|
5
|
+
uri: string;
|
|
6
|
+
cid: string;
|
|
7
|
+
recommender_did: string;
|
|
8
|
+
playlist_idea_uri: string;
|
|
9
|
+
playlist_idea_cid: string;
|
|
10
|
+
song_id: number;
|
|
11
|
+
note: string | null;
|
|
12
|
+
created_at: string;
|
|
13
|
+
indexed_at: string;
|
|
14
|
+
}
|
|
15
|
+
export type PartialDB = {
|
|
16
|
+
[tableName]: SongRecommendationTable;
|
|
17
|
+
};
|
|
@@ -4,8 +4,8 @@ export interface SongTable {
|
|
|
4
4
|
id: Generated<number>;
|
|
5
5
|
track_name: string;
|
|
6
6
|
track_mb_id: string;
|
|
7
|
-
album_artwork_ref
|
|
8
|
-
|
|
7
|
+
album_artwork_ref: string | null;
|
|
8
|
+
indexed_at: string;
|
|
9
9
|
}
|
|
10
10
|
export type PartialDB = {
|
|
11
11
|
[tableName]: SongTable;
|
|
@@ -1 +1,2 @@
|
|
|
1
|
-
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { DummyDriver,
|
|
1
|
+
import { DummyDriver, ExpressionBuilder, RawBuilder, SelectQueryBuilder, SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler } from 'kysely';
|
|
2
2
|
import { DatabaseSchema, DatabaseSchemaType } from './database-schema';
|
|
3
3
|
export declare const actorWhereClause: (actor: string) => RawBuilder<0 | 1>;
|
|
4
4
|
export declare const notSoftDeletedClause: (alias: DbRef) => RawBuilder<unknown>;
|
|
@@ -6,7 +6,7 @@ export declare const softDeleted: (actorOrRecord: {
|
|
|
6
6
|
takedownRef: string | null;
|
|
7
7
|
}) => boolean;
|
|
8
8
|
export declare const countAll: RawBuilder<number>;
|
|
9
|
-
export declare const excluded: <T>(db: DatabaseSchema, col:
|
|
9
|
+
export declare const excluded: <T>(db: DatabaseSchema, col: string) => RawBuilder<T>;
|
|
10
10
|
export declare const noMatch: RawBuilder<unknown>;
|
|
11
11
|
export declare const valuesList: (vals: unknown[]) => RawBuilder<unknown>;
|
|
12
12
|
export declare const dummyDialect: {
|
|
@@ -15,6 +15,10 @@ export declare const dummyDialect: {
|
|
|
15
15
|
createIntrospector(db: any): SqliteIntrospector;
|
|
16
16
|
createQueryCompiler(): SqliteQueryCompiler;
|
|
17
17
|
};
|
|
18
|
-
|
|
18
|
+
type DynamicRefReturn = DatabaseSchema['dynamic'] extends {
|
|
19
|
+
ref: (...args: unknown[]) => infer R;
|
|
20
|
+
} ? R : never;
|
|
21
|
+
export type DbRef = RawBuilder<unknown> | DynamicRefReturn;
|
|
19
22
|
export type Subquery = ExpressionBuilder<DatabaseSchemaType, any>;
|
|
20
23
|
export type AnyQb = SelectQueryBuilder<any, any, any>;
|
|
24
|
+
export {};
|
|
@@ -1,40 +1,48 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.dummyDialect = exports.valuesList = exports.noMatch = exports.excluded = exports.countAll = exports.softDeleted = exports.notSoftDeletedClause = exports.actorWhereClause = void 0;
|
|
4
|
+
const kysely_1 = require("kysely");
|
|
5
|
+
const actorWhereClause = (actor) => {
|
|
3
6
|
if (actor.startsWith('did:')) {
|
|
4
|
-
return sql `"actor"."did" = ${actor}`;
|
|
7
|
+
return (0, kysely_1.sql) `"actor"."did" = ${actor}`;
|
|
5
8
|
}
|
|
6
9
|
else {
|
|
7
|
-
return sql `"actor"."handle" = ${actor}`;
|
|
10
|
+
return (0, kysely_1.sql) `"actor"."handle" = ${actor}`;
|
|
8
11
|
}
|
|
9
12
|
};
|
|
13
|
+
exports.actorWhereClause = actorWhereClause;
|
|
10
14
|
// Applies to actor or record table
|
|
11
|
-
|
|
12
|
-
return sql `${alias}."takedownRef" is null`;
|
|
15
|
+
const notSoftDeletedClause = (alias) => {
|
|
16
|
+
return (0, kysely_1.sql) `${alias}."takedownRef" is null`;
|
|
13
17
|
};
|
|
14
|
-
|
|
18
|
+
exports.notSoftDeletedClause = notSoftDeletedClause;
|
|
19
|
+
const softDeleted = (actorOrRecord) => {
|
|
15
20
|
return actorOrRecord.takedownRef !== null;
|
|
16
21
|
};
|
|
17
|
-
|
|
22
|
+
exports.softDeleted = softDeleted;
|
|
23
|
+
exports.countAll = (0, kysely_1.sql) `count(*)`;
|
|
18
24
|
// For use with doUpdateSet()
|
|
19
|
-
|
|
20
|
-
return sql `${db.dynamic.ref(`excluded.${col}`)}`;
|
|
25
|
+
const excluded = (db, col) => {
|
|
26
|
+
return (0, kysely_1.sql) `${db.dynamic.ref(`excluded.${col}`)}`;
|
|
21
27
|
};
|
|
22
|
-
|
|
28
|
+
exports.excluded = excluded;
|
|
29
|
+
exports.noMatch = (0, kysely_1.sql) `1 = 0`;
|
|
23
30
|
// Can be useful for large where-in clauses, to get the db to use a hash lookup on the list
|
|
24
|
-
|
|
25
|
-
return sql `(values (${sql.join(vals, sql `), (`)}))`;
|
|
31
|
+
const valuesList = (vals) => {
|
|
32
|
+
return (0, kysely_1.sql) `(values (${kysely_1.sql.join(vals, (0, kysely_1.sql) `), (`)}))`;
|
|
26
33
|
};
|
|
27
|
-
|
|
34
|
+
exports.valuesList = valuesList;
|
|
35
|
+
exports.dummyDialect = {
|
|
28
36
|
createAdapter() {
|
|
29
|
-
return new SqliteAdapter();
|
|
37
|
+
return new kysely_1.SqliteAdapter();
|
|
30
38
|
},
|
|
31
39
|
createDriver() {
|
|
32
|
-
return new DummyDriver();
|
|
40
|
+
return new kysely_1.DummyDriver();
|
|
33
41
|
},
|
|
34
42
|
createIntrospector(db) {
|
|
35
|
-
return new SqliteIntrospector(db);
|
|
43
|
+
return new kysely_1.SqliteIntrospector(db);
|
|
36
44
|
},
|
|
37
45
|
createQueryCompiler() {
|
|
38
|
-
return new SqliteQueryCompiler();
|
|
46
|
+
return new kysely_1.SqliteQueryCompiler();
|
|
39
47
|
},
|
|
40
48
|
};
|
|
@@ -1,10 +1,17 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.DataPlaneServer = exports.RepoSubscription = void 0;
|
|
7
|
+
const identity_1 = require("@atproto/identity");
|
|
8
|
+
const connect_express_1 = require("@connectrpc/connect-express");
|
|
9
|
+
const express_1 = __importDefault(require("express"));
|
|
10
|
+
const node_events_1 = __importDefault(require("node:events"));
|
|
11
|
+
const routes_1 = __importDefault(require("./routes"));
|
|
12
|
+
var subscription_1 = require("./subscription");
|
|
13
|
+
Object.defineProperty(exports, "RepoSubscription", { enumerable: true, get: function () { return subscription_1.RepoSubscription; } });
|
|
14
|
+
class DataPlaneServer {
|
|
8
15
|
server;
|
|
9
16
|
idResolver;
|
|
10
17
|
constructor(server, idResolver) {
|
|
@@ -12,13 +19,13 @@ export class DataPlaneServer {
|
|
|
12
19
|
this.idResolver = idResolver;
|
|
13
20
|
}
|
|
14
21
|
static async create(db, port, plcUrl) {
|
|
15
|
-
const app =
|
|
16
|
-
const didCache = new MemoryCache();
|
|
17
|
-
const idResolver = new IdResolver({ plcUrl, didCache });
|
|
18
|
-
const routes =
|
|
19
|
-
app.use(expressConnectMiddleware({ routes }));
|
|
22
|
+
const app = (0, express_1.default)();
|
|
23
|
+
const didCache = new identity_1.MemoryCache();
|
|
24
|
+
const idResolver = new identity_1.IdResolver({ plcUrl, didCache });
|
|
25
|
+
const routes = (0, routes_1.default)(db, idResolver);
|
|
26
|
+
app.use((0, connect_express_1.expressConnectMiddleware)({ routes }));
|
|
20
27
|
const server = app.listen(port);
|
|
21
|
-
await
|
|
28
|
+
await node_events_1.default.once(server, 'listening');
|
|
22
29
|
return new DataPlaneServer(server, idResolver);
|
|
23
30
|
}
|
|
24
31
|
async destroy() {
|
|
@@ -34,3 +41,4 @@ export class DataPlaneServer {
|
|
|
34
41
|
});
|
|
35
42
|
}
|
|
36
43
|
}
|
|
44
|
+
exports.DataPlaneServer = DataPlaneServer;
|
|
@@ -5,6 +5,7 @@ import { CID } from 'multiformats/cid';
|
|
|
5
5
|
import { Database } from '../db';
|
|
6
6
|
import * as PlaylistIdea from './plugins/playlist-idea';
|
|
7
7
|
import * as Profile from './plugins/profile';
|
|
8
|
+
import * as SongRecommendation from './plugins/song-recommendation';
|
|
8
9
|
import { RecordProcessor } from './processor';
|
|
9
10
|
export declare class IndexingService {
|
|
10
11
|
db: Database;
|
|
@@ -12,6 +13,7 @@ export declare class IndexingService {
|
|
|
12
13
|
records: {
|
|
13
14
|
playlistIdea: PlaylistIdea.PluginType;
|
|
14
15
|
profile: Profile.PluginType;
|
|
16
|
+
songRecommendation: SongRecommendation.PluginType;
|
|
15
17
|
};
|
|
16
18
|
constructor(db: Database, idResolver: IdResolver);
|
|
17
19
|
transact(txn: Database): IndexingService;
|
|
@@ -1,16 +1,53 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.IndexingService = void 0;
|
|
37
|
+
const api_1 = require("@atproto/api");
|
|
38
|
+
const common_1 = require("@atproto/common");
|
|
39
|
+
const identity_1 = require("@atproto/identity");
|
|
40
|
+
const lexicon_1 = require("@atproto/lexicon");
|
|
41
|
+
const repo_1 = require("@atproto/repo");
|
|
42
|
+
const syntax_1 = require("@atproto/syntax");
|
|
43
|
+
const kysely_1 = require("kysely");
|
|
44
|
+
const cid_1 = require("multiformats/cid");
|
|
45
|
+
const logger_1 = require("../../../logger");
|
|
46
|
+
const retry_1 = require("../../../util/retry");
|
|
47
|
+
const PlaylistIdea = __importStar(require("./plugins/playlist-idea"));
|
|
48
|
+
const Profile = __importStar(require("./plugins/profile"));
|
|
49
|
+
const SongRecommendation = __importStar(require("./plugins/song-recommendation"));
|
|
50
|
+
class IndexingService {
|
|
14
51
|
db;
|
|
15
52
|
idResolver;
|
|
16
53
|
records;
|
|
@@ -20,6 +57,7 @@ export class IndexingService {
|
|
|
20
57
|
this.records = {
|
|
21
58
|
playlistIdea: PlaylistIdea.makePlugin(this.db),
|
|
22
59
|
profile: Profile.makePlugin(this.db),
|
|
60
|
+
songRecommendation: SongRecommendation.makePlugin(this.db),
|
|
23
61
|
};
|
|
24
62
|
}
|
|
25
63
|
transact(txn) {
|
|
@@ -33,7 +71,7 @@ export class IndexingService {
|
|
|
33
71
|
const indexer = indexingTx.findIndexerForCollection(uri.collection);
|
|
34
72
|
if (!indexer)
|
|
35
73
|
return;
|
|
36
|
-
if (action === WriteOpAction.Create) {
|
|
74
|
+
if (action === repo_1.WriteOpAction.Create) {
|
|
37
75
|
await indexer.insertRecord(uri, cid, obj, timestamp);
|
|
38
76
|
}
|
|
39
77
|
else {
|
|
@@ -53,11 +91,7 @@ export class IndexingService {
|
|
|
53
91
|
}
|
|
54
92
|
async indexHandle(did, timestamp, force = false) {
|
|
55
93
|
this.db.assertNotTransaction();
|
|
56
|
-
const actor = await this.db.db
|
|
57
|
-
.selectFrom('actor')
|
|
58
|
-
.where('did', '=', did)
|
|
59
|
-
.selectAll()
|
|
60
|
-
.executeTakeFirst();
|
|
94
|
+
const actor = await this.db.db.selectFrom('actor').where('did', '=', did).selectAll().executeTakeFirst();
|
|
61
95
|
if (!force && !needsHandleReindex(actor, timestamp)) {
|
|
62
96
|
return;
|
|
63
97
|
}
|
|
@@ -65,19 +99,11 @@ export class IndexingService {
|
|
|
65
99
|
const handleToDid = await this.idResolver.handle.resolve(atpData.handle);
|
|
66
100
|
const handle = did === handleToDid ? atpData.handle.toLowerCase() : null;
|
|
67
101
|
const actorWithHandle = handle !== null
|
|
68
|
-
? await this.db.db
|
|
69
|
-
.selectFrom('actor')
|
|
70
|
-
.where('handle', '=', handle)
|
|
71
|
-
.selectAll()
|
|
72
|
-
.executeTakeFirst()
|
|
102
|
+
? await this.db.db.selectFrom('actor').where('handle', '=', handle).selectAll().executeTakeFirst()
|
|
73
103
|
: null;
|
|
74
104
|
// handle contention
|
|
75
105
|
if (handle && actorWithHandle && did !== actorWithHandle.did) {
|
|
76
|
-
await this.db.db
|
|
77
|
-
.updateTable('actor')
|
|
78
|
-
.where('actor.did', '=', actorWithHandle.did)
|
|
79
|
-
.set({ handle: null })
|
|
80
|
-
.execute();
|
|
106
|
+
await this.db.db.updateTable('actor').where('actor.did', '=', actorWithHandle.did).set({ handle: null }).execute();
|
|
81
107
|
}
|
|
82
108
|
const actorInfo = { handle, indexed_at: timestamp };
|
|
83
109
|
await this.db.db
|
|
@@ -91,10 +117,10 @@ export class IndexingService {
|
|
|
91
117
|
this.db.assertNotTransaction();
|
|
92
118
|
const now = new Date().toISOString();
|
|
93
119
|
const { pds, signingKey } = await this.idResolver.did.resolveAtprotoData(did, true);
|
|
94
|
-
const agent = new Agent(pds);
|
|
95
|
-
const { data: car } = await retryXrpc(() => agent.com.atproto.sync.getRepo({ did }));
|
|
96
|
-
const { root, blocks } = await readCarWithRoot(car);
|
|
97
|
-
const verifiedRepo = await verifyRepo(blocks, root, did, signingKey);
|
|
120
|
+
const agent = new api_1.Agent(pds);
|
|
121
|
+
const { data: car } = await (0, retry_1.retryXrpc)(() => agent.com.atproto.sync.getRepo({ did }));
|
|
122
|
+
const { root, blocks } = await (0, repo_1.readCarWithRoot)(car);
|
|
123
|
+
const verifiedRepo = await (0, repo_1.verifyRepo)(blocks, root, did, signingKey);
|
|
98
124
|
const currRecords = await this.getCurrentRecords(did);
|
|
99
125
|
const repoRecords = formatCheckout(did, verifiedRepo);
|
|
100
126
|
const diff = findDiffFromCheckout(currRecords, repoRecords);
|
|
@@ -105,16 +131,16 @@ export class IndexingService {
|
|
|
105
131
|
await this.deleteRecord(uri);
|
|
106
132
|
}
|
|
107
133
|
else {
|
|
108
|
-
const parsed = await getAndParseRecord(blocks, cid);
|
|
109
|
-
await this.indexRecord(uri, cid, parsed.record, op.op === 'create' ? WriteOpAction.Create : WriteOpAction.Update, now);
|
|
134
|
+
const parsed = await (0, repo_1.getAndParseRecord)(blocks, cid);
|
|
135
|
+
await this.indexRecord(uri, cid, parsed.record, op.op === 'create' ? repo_1.WriteOpAction.Create : repo_1.WriteOpAction.Update, now);
|
|
110
136
|
}
|
|
111
137
|
}
|
|
112
138
|
catch (err) {
|
|
113
|
-
if (err instanceof ValidationError) {
|
|
114
|
-
httpLogger.warn({ did, commit, uri: uri.toString(), cid: cid.toString() }, 'skipping indexing of invalid record');
|
|
139
|
+
if (err instanceof lexicon_1.ValidationError) {
|
|
140
|
+
logger_1.httpLogger.warn({ did, commit, uri: uri.toString(), cid: cid.toString() }, 'skipping indexing of invalid record');
|
|
115
141
|
}
|
|
116
142
|
else {
|
|
117
|
-
httpLogger.error({ err, did, commit, uri: uri.toString(), cid: cid.toString() }, 'skipping indexing due to error processing record');
|
|
143
|
+
logger_1.httpLogger.error({ err, did, commit, uri: uri.toString(), cid: cid.toString() }, 'skipping indexing due to error processing record');
|
|
118
144
|
}
|
|
119
145
|
}
|
|
120
146
|
}));
|
|
@@ -130,11 +156,7 @@ export class IndexingService {
|
|
|
130
156
|
else {
|
|
131
157
|
throw new Error(`Unrecognized account status: ${status}`);
|
|
132
158
|
}
|
|
133
|
-
await this.db.db
|
|
134
|
-
.updateTable('actor')
|
|
135
|
-
.set({ upstreamStatus })
|
|
136
|
-
.where('did', '=', did)
|
|
137
|
-
.execute();
|
|
159
|
+
await this.db.db.updateTable('actor').set({ upstream_status: upstreamStatus }).where('did', '=', did).execute();
|
|
138
160
|
}
|
|
139
161
|
async setCommitLastSeen(did, commit, rev) {
|
|
140
162
|
const { ref } = this.db.db.dynamic;
|
|
@@ -142,28 +164,24 @@ export class IndexingService {
|
|
|
142
164
|
.insertInto('actor_sync')
|
|
143
165
|
.values({
|
|
144
166
|
did,
|
|
145
|
-
|
|
146
|
-
|
|
167
|
+
commit_cid: commit.toString(),
|
|
168
|
+
repo_rev: rev,
|
|
147
169
|
})
|
|
148
170
|
.onConflict((oc) => {
|
|
149
171
|
const excluded = (col) => ref(`excluded.${col}`);
|
|
150
172
|
return oc.column('did').doUpdateSet({
|
|
151
|
-
|
|
152
|
-
|
|
173
|
+
commit_cid: (0, kysely_1.sql) `${excluded('commit_cid')}`,
|
|
174
|
+
repo_rev: (0, kysely_1.sql) `${excluded('repo_rev')}`,
|
|
153
175
|
});
|
|
154
176
|
})
|
|
155
177
|
.execute();
|
|
156
178
|
}
|
|
157
179
|
async getCurrentRecords(did) {
|
|
158
|
-
const res = await this.db.db
|
|
159
|
-
.selectFrom('record')
|
|
160
|
-
.where('did', '=', did)
|
|
161
|
-
.select(['uri', 'cid'])
|
|
162
|
-
.execute();
|
|
180
|
+
const res = await this.db.db.selectFrom('record').where('did', '=', did).select(['uri', 'cid']).execute();
|
|
163
181
|
return res.reduce((acc, cur) => {
|
|
164
182
|
acc[cur.uri] = {
|
|
165
|
-
uri: new AtUri(cur.uri),
|
|
166
|
-
cid: CID.parse(cur.cid),
|
|
183
|
+
uri: new syntax_1.AtUri(cur.uri),
|
|
184
|
+
cid: cid_1.CID.parse(cur.cid),
|
|
167
185
|
};
|
|
168
186
|
return acc;
|
|
169
187
|
}, {});
|
|
@@ -182,16 +200,16 @@ export class IndexingService {
|
|
|
182
200
|
}
|
|
183
201
|
async getActorIsHosted(did) {
|
|
184
202
|
const doc = await this.idResolver.did.resolve(did, true);
|
|
185
|
-
const pds = doc && getPds(doc);
|
|
203
|
+
const pds = doc && (0, identity_1.getPds)(doc);
|
|
186
204
|
if (!pds)
|
|
187
205
|
return false;
|
|
188
|
-
const agent = new Agent(pds);
|
|
206
|
+
const agent = new api_1.Agent(pds);
|
|
189
207
|
try {
|
|
190
|
-
await retryXrpc(() => agent.com.atproto.sync.getLatestCommit({ did }));
|
|
208
|
+
await (0, retry_1.retryXrpc)(() => agent.com.atproto.sync.getLatestCommit({ did }));
|
|
191
209
|
return true;
|
|
192
210
|
}
|
|
193
211
|
catch (err) {
|
|
194
|
-
if (err instanceof ComAtprotoSyncGetLatestCommit.RepoNotFoundError) {
|
|
212
|
+
if (err instanceof api_1.ComAtprotoSyncGetLatestCommit.RepoNotFoundError) {
|
|
195
213
|
return false;
|
|
196
214
|
}
|
|
197
215
|
return null;
|
|
@@ -199,7 +217,7 @@ export class IndexingService {
|
|
|
199
217
|
}
|
|
200
218
|
async unindexActor(did) {
|
|
201
219
|
this.db.assertNotTransaction();
|
|
202
|
-
await this.db.db.deleteFrom('profile').where('
|
|
220
|
+
await this.db.db.deleteFrom('profile').where('creator_did', '=', did).execute();
|
|
203
221
|
await this.db.db.deleteFrom('record').where('did', '=', did).execute();
|
|
204
222
|
const deletedPlaylistIdeas = await this.db.db
|
|
205
223
|
.deleteFrom('playlist_idea')
|
|
@@ -217,12 +235,10 @@ export class IndexingService {
|
|
|
217
235
|
const deletedPlaylistIds = deletedPlaylists.map((deletedPlaylist) => {
|
|
218
236
|
return deletedPlaylist.id;
|
|
219
237
|
});
|
|
220
|
-
await this.db.db
|
|
221
|
-
.deleteFrom('playlist_item')
|
|
222
|
-
.where('playlist_id', 'in', deletedPlaylistIds)
|
|
223
|
-
.execute();
|
|
238
|
+
await this.db.db.deleteFrom('playlist_item').where('playlist_id', 'in', deletedPlaylistIds).execute();
|
|
224
239
|
}
|
|
225
240
|
}
|
|
241
|
+
exports.IndexingService = IndexingService;
|
|
226
242
|
const findDiffFromCheckout = (curr, checkout) => {
|
|
227
243
|
const ops = [];
|
|
228
244
|
for (const uri of Object.keys(checkout)) {
|
|
@@ -249,7 +265,7 @@ const findDiffFromCheckout = (curr, checkout) => {
|
|
|
249
265
|
const formatCheckout = (did, verifiedRepo) => {
|
|
250
266
|
const records = {};
|
|
251
267
|
for (const create of verifiedRepo.creates) {
|
|
252
|
-
const uri = AtUri.make(did, create.collection, create.rkey);
|
|
268
|
+
const uri = syntax_1.AtUri.make(did, create.collection, create.rkey);
|
|
253
269
|
records[uri.toString()] = {
|
|
254
270
|
uri,
|
|
255
271
|
cid: create.cid,
|
|
@@ -262,10 +278,10 @@ const needsHandleReindex = (actor, timestamp) => {
|
|
|
262
278
|
return true;
|
|
263
279
|
const timeDiff = new Date(timestamp).getTime() - new Date(actor.indexed_at).getTime();
|
|
264
280
|
// revalidate daily
|
|
265
|
-
if (timeDiff > DAY)
|
|
281
|
+
if (timeDiff > common_1.DAY)
|
|
266
282
|
return true;
|
|
267
283
|
// revalidate more aggressively for invalidated handles
|
|
268
|
-
if (actor.handle === null && timeDiff > HOUR)
|
|
284
|
+
if (actor.handle === null && timeDiff > common_1.HOUR)
|
|
269
285
|
return true;
|
|
270
286
|
return false;
|
|
271
287
|
};
|