@clioplaylists/clio 0.1.6 → 0.1.7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.env +7 -0
- package/dist/api/health.js +3 -10
- package/dist/api/index.js +2 -39
- package/dist/api/util.js +6 -10
- package/dist/auth-verifier.js +165 -273
- package/dist/client.js +8 -15
- package/dist/config.js +7 -23
- package/dist/context.js +3 -10
- package/dist/dataplane/client.js +25 -70
- package/dist/dataplane/index.js +2 -18
- package/dist/dataplane/server/background.js +8 -29
- package/dist/dataplane/server/db/database-schema.js +1 -2
- package/dist/dataplane/server/db/db.js +27 -103
- package/dist/dataplane/server/db/index.js +1 -17
- package/dist/dataplane/server/db/migrations/20230309T045948368Z-init.js +7 -11
- package/dist/dataplane/server/db/migrations/20230420T211446071Z-did-cache.js +2 -6
- package/dist/dataplane/server/db/migrations/index.js +2 -38
- package/dist/dataplane/server/db/migrations/provider.js +5 -17
- package/dist/dataplane/server/db/pagination.js +21 -37
- package/dist/dataplane/server/db/tables/actor-sync.js +1 -4
- package/dist/dataplane/server/db/tables/actor.js +1 -4
- package/dist/dataplane/server/db/tables/artist-list-item.js +1 -4
- package/dist/dataplane/server/db/tables/artist.js +1 -4
- package/dist/dataplane/server/db/tables/playlist-idea.js +1 -4
- package/dist/dataplane/server/db/tables/playlist-item.js +1 -4
- package/dist/dataplane/server/db/tables/playlist.js +1 -4
- package/dist/dataplane/server/db/tables/profile.js +1 -4
- package/dist/dataplane/server/db/tables/record.js +1 -4
- package/dist/dataplane/server/db/tables/song.js +1 -4
- package/dist/dataplane/server/db/types.js +1 -2
- package/dist/dataplane/server/db/util.js +18 -26
- package/dist/dataplane/server/index.js +17 -33
- package/dist/dataplane/server/indexing/index.js +39 -89
- package/dist/dataplane/server/indexing/plugins/playlist-idea.js +7 -44
- package/dist/dataplane/server/indexing/plugins/profile.js +5 -42
- package/dist/dataplane/server/indexing/processor.js +11 -29
- package/dist/dataplane/server/routes/identity.js +11 -13
- package/dist/dataplane/server/routes/index.js +10 -15
- package/dist/dataplane/server/routes/profile.js +9 -11
- package/dist/dataplane/server/routes/records.js +14 -51
- package/dist/dataplane/server/routes/sync.js +1 -3
- package/dist/dataplane/server/subscription.js +19 -47
- package/dist/error.js +5 -9
- package/dist/index.js +29 -89
- package/dist/lexicons/index.js +52 -227
- package/dist/lexicons/lexicons.js +5 -8
- package/dist/lexicons/types/com/atproto/admin/defs.js +27 -39
- package/dist/lexicons/types/com/atproto/admin/deleteAccount.js +1 -2
- package/dist/lexicons/types/com/atproto/admin/disableAccountInvites.js +1 -2
- package/dist/lexicons/types/com/atproto/admin/disableInviteCodes.js +1 -2
- package/dist/lexicons/types/com/atproto/admin/enableAccountInvites.js +1 -2
- package/dist/lexicons/types/com/atproto/admin/getAccountInfo.js +1 -2
- package/dist/lexicons/types/com/atproto/admin/getAccountInfos.js +1 -2
- package/dist/lexicons/types/com/atproto/admin/getInviteCodes.js +1 -2
- package/dist/lexicons/types/com/atproto/admin/getSubjectStatus.js +1 -2
- package/dist/lexicons/types/com/atproto/admin/searchAccounts.js +1 -2
- package/dist/lexicons/types/com/atproto/admin/sendEmail.js +1 -2
- package/dist/lexicons/types/com/atproto/admin/updateAccountEmail.js +1 -2
- package/dist/lexicons/types/com/atproto/admin/updateAccountHandle.js +1 -2
- package/dist/lexicons/types/com/atproto/admin/updateAccountPassword.js +1 -2
- package/dist/lexicons/types/com/atproto/admin/updateSubjectStatus.js +1 -2
- package/dist/lexicons/types/com/atproto/identity/defs.js +7 -11
- package/dist/lexicons/types/com/atproto/identity/getRecommendedDidCredentials.js +1 -2
- package/dist/lexicons/types/com/atproto/identity/refreshIdentity.js +1 -2
- package/dist/lexicons/types/com/atproto/identity/requestPlcOperationSignature.js +1 -2
- package/dist/lexicons/types/com/atproto/identity/resolveDid.js +1 -2
- package/dist/lexicons/types/com/atproto/identity/resolveHandle.js +1 -2
- package/dist/lexicons/types/com/atproto/identity/resolveIdentity.js +1 -2
- package/dist/lexicons/types/com/atproto/identity/signPlcOperation.js +1 -2
- package/dist/lexicons/types/com/atproto/identity/submitPlcOperation.js +1 -2
- package/dist/lexicons/types/com/atproto/identity/updateHandle.js +1 -2
- package/dist/lexicons/types/com/atproto/label/defs.js +27 -39
- package/dist/lexicons/types/com/atproto/label/queryLabels.js +1 -2
- package/dist/lexicons/types/com/atproto/label/subscribeLabels.js +12 -18
- package/dist/lexicons/types/com/atproto/lexicon/schema.js +7 -11
- package/dist/lexicons/types/com/atproto/moderation/createReport.js +1 -2
- package/dist/lexicons/types/com/atproto/moderation/defs.js +7 -10
- package/dist/lexicons/types/com/atproto/repo/applyWrites.js +32 -46
- package/dist/lexicons/types/com/atproto/repo/createRecord.js +1 -2
- package/dist/lexicons/types/com/atproto/repo/defs.js +7 -11
- package/dist/lexicons/types/com/atproto/repo/deleteRecord.js +1 -2
- package/dist/lexicons/types/com/atproto/repo/describeRepo.js +1 -2
- package/dist/lexicons/types/com/atproto/repo/getRecord.js +1 -2
- package/dist/lexicons/types/com/atproto/repo/importRepo.js +1 -2
- package/dist/lexicons/types/com/atproto/repo/listMissingBlobs.js +7 -11
- package/dist/lexicons/types/com/atproto/repo/listRecords.js +7 -11
- package/dist/lexicons/types/com/atproto/repo/putRecord.js +1 -2
- package/dist/lexicons/types/com/atproto/repo/strongRef.js +7 -11
- package/dist/lexicons/types/com/atproto/repo/uploadBlob.js +1 -2
- package/dist/lexicons/types/com/atproto/server/activateAccount.js +1 -2
- package/dist/lexicons/types/com/atproto/server/checkAccountStatus.js +1 -2
- package/dist/lexicons/types/com/atproto/server/confirmEmail.js +1 -2
- package/dist/lexicons/types/com/atproto/server/createAccount.js +1 -2
- package/dist/lexicons/types/com/atproto/server/createAppPassword.js +7 -11
- package/dist/lexicons/types/com/atproto/server/createInviteCode.js +1 -2
- package/dist/lexicons/types/com/atproto/server/createInviteCodes.js +7 -11
- package/dist/lexicons/types/com/atproto/server/createSession.js +1 -2
- package/dist/lexicons/types/com/atproto/server/deactivateAccount.js +1 -2
- package/dist/lexicons/types/com/atproto/server/defs.js +12 -18
- package/dist/lexicons/types/com/atproto/server/deleteAccount.js +1 -2
- package/dist/lexicons/types/com/atproto/server/deleteSession.js +1 -2
- package/dist/lexicons/types/com/atproto/server/describeServer.js +12 -18
- package/dist/lexicons/types/com/atproto/server/getAccountInviteCodes.js +1 -2
- package/dist/lexicons/types/com/atproto/server/getServiceAuth.js +1 -2
- package/dist/lexicons/types/com/atproto/server/getSession.js +1 -2
- package/dist/lexicons/types/com/atproto/server/listAppPasswords.js +7 -11
- package/dist/lexicons/types/com/atproto/server/refreshSession.js +1 -2
- package/dist/lexicons/types/com/atproto/server/requestAccountDelete.js +1 -2
- package/dist/lexicons/types/com/atproto/server/requestEmailConfirmation.js +1 -2
- package/dist/lexicons/types/com/atproto/server/requestEmailUpdate.js +1 -2
- package/dist/lexicons/types/com/atproto/server/requestPasswordReset.js +1 -2
- package/dist/lexicons/types/com/atproto/server/reserveSigningKey.js +1 -2
- package/dist/lexicons/types/com/atproto/server/resetPassword.js +1 -2
- package/dist/lexicons/types/com/atproto/server/revokeAppPassword.js +1 -2
- package/dist/lexicons/types/com/atproto/server/updateEmail.js +1 -2
- package/dist/lexicons/types/com/atproto/sync/getBlob.js +1 -2
- package/dist/lexicons/types/com/atproto/sync/getBlocks.js +1 -2
- package/dist/lexicons/types/com/atproto/sync/getCheckout.js +1 -2
- package/dist/lexicons/types/com/atproto/sync/getHead.js +1 -2
- package/dist/lexicons/types/com/atproto/sync/getLatestCommit.js +1 -2
- package/dist/lexicons/types/com/atproto/sync/getRecord.js +1 -2
- package/dist/lexicons/types/com/atproto/sync/getRepo.js +1 -2
- package/dist/lexicons/types/com/atproto/sync/getRepoStatus.js +1 -2
- package/dist/lexicons/types/com/atproto/sync/listBlobs.js +1 -2
- package/dist/lexicons/types/com/atproto/sync/listRepos.js +7 -11
- package/dist/lexicons/types/com/atproto/sync/listReposByCollection.js +7 -11
- package/dist/lexicons/types/com/atproto/sync/notifyOfUpdate.js +1 -2
- package/dist/lexicons/types/com/atproto/sync/requestCrawl.js +1 -2
- package/dist/lexicons/types/com/atproto/sync/subscribeRepos.js +32 -46
- package/dist/lexicons/types/com/atproto/temp/addReservedHandle.js +1 -2
- package/dist/lexicons/types/com/atproto/temp/checkSignupQueue.js +1 -2
- package/dist/lexicons/types/com/atproto/temp/fetchLabels.js +1 -2
- package/dist/lexicons/types/com/atproto/temp/requestPhoneVerification.js +1 -2
- package/dist/lexicons/types/com/clioplaylists/alpha/actor/profile.js +7 -11
- package/dist/lexicons/types/com/clioplaylists/alpha/feed/defs.js +12 -18
- package/dist/lexicons/types/com/clioplaylists/alpha/feed/getSongs.js +1 -2
- package/dist/lexicons/types/com/clioplaylists/alpha/feed/playlistIdea.js +17 -25
- package/dist/lexicons/util.js +2 -6
- package/dist/logger.js +10 -16
- package/dist/rpc/clio_connect.js +30 -33
- package/dist/rpc/clio_pb.js +402 -956
- package/dist/util/retry.js +7 -10
- package/dist/util/uris.js +3 -6
- package/dist/util.js +17 -60
- package/package.json +4 -4
|
@@ -1,8 +1,5 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.paginate = exports.IndexedAtDidKeyset = exports.CreatedAtDidKeyset = exports.TimeCidKeyset = exports.GenericKeyset = void 0;
|
|
4
|
-
const xrpc_server_1 = require("@atproto/xrpc-server");
|
|
5
|
-
const kysely_1 = require("kysely");
|
|
1
|
+
import { InvalidRequestError } from '@atproto/xrpc-server';
|
|
2
|
+
import { sql } from 'kysely';
|
|
6
3
|
/**
|
|
7
4
|
* The GenericKeyset is an abstract class that sets-up the interface and partial implementation
|
|
8
5
|
* of a keyset-paginated cursor with two parts. There are three types involved:
|
|
@@ -17,20 +14,12 @@ const kysely_1 = require("kysely");
|
|
|
17
14
|
* Result -*-> LabeledResult <-*-> Cursor <--> packed/string cursor
|
|
18
15
|
* ↳ SQL Condition
|
|
19
16
|
*/
|
|
20
|
-
class GenericKeyset {
|
|
17
|
+
export class GenericKeyset {
|
|
18
|
+
primary;
|
|
19
|
+
secondary;
|
|
21
20
|
constructor(primary, secondary) {
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
configurable: true,
|
|
25
|
-
writable: true,
|
|
26
|
-
value: primary
|
|
27
|
-
});
|
|
28
|
-
Object.defineProperty(this, "secondary", {
|
|
29
|
-
enumerable: true,
|
|
30
|
-
configurable: true,
|
|
31
|
-
writable: true,
|
|
32
|
-
value: secondary
|
|
33
|
-
});
|
|
21
|
+
this.primary = primary;
|
|
22
|
+
this.secondary = secondary;
|
|
34
23
|
}
|
|
35
24
|
packFromResult(results) {
|
|
36
25
|
const result = Array.isArray(results) ? results.at(-1) : results;
|
|
@@ -61,7 +50,7 @@ class GenericKeyset {
|
|
|
61
50
|
const result = cursorStr.split('__');
|
|
62
51
|
const [primary, secondary, ...others] = result;
|
|
63
52
|
if (!primary || !secondary || others.length > 0) {
|
|
64
|
-
throw new
|
|
53
|
+
throw new InvalidRequestError('Malformed cursor');
|
|
65
54
|
}
|
|
66
55
|
return {
|
|
67
56
|
primary,
|
|
@@ -74,25 +63,24 @@ class GenericKeyset {
|
|
|
74
63
|
if (tryIndex) {
|
|
75
64
|
// The tryIndex param will likely disappear and become the default implementation: here for now for gradual rollout query-by-query.
|
|
76
65
|
if (direction === 'asc') {
|
|
77
|
-
return
|
|
66
|
+
return sql `((${this.primary}, ${this.secondary}) > (${labeled.primary}, ${labeled.secondary}))`;
|
|
78
67
|
}
|
|
79
68
|
else {
|
|
80
|
-
return
|
|
69
|
+
return sql `((${this.primary}, ${this.secondary}) < (${labeled.primary}, ${labeled.secondary}))`;
|
|
81
70
|
}
|
|
82
71
|
}
|
|
83
72
|
else {
|
|
84
73
|
// @NOTE this implementation can struggle to use an index on (primary, secondary) for pagination due to the "or" usage.
|
|
85
74
|
if (direction === 'asc') {
|
|
86
|
-
return
|
|
75
|
+
return sql `((${this.primary} > ${labeled.primary}) or (${this.primary} = ${labeled.primary} and ${this.secondary} > ${labeled.secondary}))`;
|
|
87
76
|
}
|
|
88
77
|
else {
|
|
89
|
-
return
|
|
78
|
+
return sql `((${this.primary} < ${labeled.primary}) or (${this.primary} = ${labeled.primary} and ${this.secondary} < ${labeled.secondary}))`;
|
|
90
79
|
}
|
|
91
80
|
}
|
|
92
81
|
}
|
|
93
82
|
}
|
|
94
|
-
|
|
95
|
-
class TimeCidKeyset extends GenericKeyset {
|
|
83
|
+
export class TimeCidKeyset extends GenericKeyset {
|
|
96
84
|
labelResult(result) {
|
|
97
85
|
return { primary: result.sortAt, secondary: result.cid };
|
|
98
86
|
}
|
|
@@ -105,7 +93,7 @@ class TimeCidKeyset extends GenericKeyset {
|
|
|
105
93
|
cursorToLabeledResult(cursor) {
|
|
106
94
|
const primaryDate = new Date(parseInt(cursor.primary, 10));
|
|
107
95
|
if (isNaN(primaryDate.getTime())) {
|
|
108
|
-
throw new
|
|
96
|
+
throw new InvalidRequestError('Malformed cursor');
|
|
109
97
|
}
|
|
110
98
|
return {
|
|
111
99
|
primary: primaryDate.toISOString(),
|
|
@@ -113,20 +101,17 @@ class TimeCidKeyset extends GenericKeyset {
|
|
|
113
101
|
};
|
|
114
102
|
}
|
|
115
103
|
}
|
|
116
|
-
|
|
117
|
-
class CreatedAtDidKeyset extends TimeCidKeyset {
|
|
104
|
+
export class CreatedAtDidKeyset extends TimeCidKeyset {
|
|
118
105
|
labelResult(result) {
|
|
119
106
|
return { primary: result.createdAt, secondary: result.did };
|
|
120
107
|
}
|
|
121
108
|
}
|
|
122
|
-
|
|
123
|
-
class IndexedAtDidKeyset extends TimeCidKeyset {
|
|
109
|
+
export class IndexedAtDidKeyset extends TimeCidKeyset {
|
|
124
110
|
labelResult(result) {
|
|
125
111
|
return { primary: result.indexedAt, secondary: result.did };
|
|
126
112
|
}
|
|
127
113
|
}
|
|
128
|
-
|
|
129
|
-
const paginate = (qb, opts) => {
|
|
114
|
+
export const paginate = (qb, opts) => {
|
|
130
115
|
const { limit, cursor, keyset, direction = 'desc', tryIndex, nullsLast, } = opts;
|
|
131
116
|
const keysetSql = keyset.getSql(keyset.unpack(cursor), direction, tryIndex);
|
|
132
117
|
return qb
|
|
@@ -134,11 +119,10 @@ const paginate = (qb, opts) => {
|
|
|
134
119
|
.if(!nullsLast, (q) => q.orderBy(keyset.primary, direction).orderBy(keyset.secondary, direction))
|
|
135
120
|
.if(!!nullsLast, (q) => q
|
|
136
121
|
.orderBy(direction === 'asc'
|
|
137
|
-
?
|
|
138
|
-
:
|
|
122
|
+
? sql `${keyset.primary} asc nulls last`
|
|
123
|
+
: sql `${keyset.primary} desc nulls last`)
|
|
139
124
|
.orderBy(direction === 'asc'
|
|
140
|
-
?
|
|
141
|
-
:
|
|
125
|
+
? sql `${keyset.secondary} asc nulls last`
|
|
126
|
+
: sql `${keyset.secondary} desc nulls last`))
|
|
142
127
|
.if(!!keysetSql, (qb) => (keysetSql ? qb.where(keysetSql) : qb));
|
|
143
128
|
};
|
|
144
|
-
exports.paginate = paginate;
|
|
@@ -1,2 +1 @@
|
|
|
1
|
-
|
|
2
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
1
|
+
export {};
|
|
@@ -1,48 +1,40 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
exports.dummyDialect = exports.valuesList = exports.noMatch = exports.excluded = exports.countAll = exports.softDeleted = exports.notSoftDeletedClause = exports.actorWhereClause = void 0;
|
|
4
|
-
const kysely_1 = require("kysely");
|
|
5
|
-
const actorWhereClause = (actor) => {
|
|
1
|
+
import { DummyDriver, SqliteAdapter, SqliteIntrospector, SqliteQueryCompiler, sql, } from 'kysely';
|
|
2
|
+
export const actorWhereClause = (actor) => {
|
|
6
3
|
if (actor.startsWith('did:')) {
|
|
7
|
-
return
|
|
4
|
+
return sql `"actor"."did" = ${actor}`;
|
|
8
5
|
}
|
|
9
6
|
else {
|
|
10
|
-
return
|
|
7
|
+
return sql `"actor"."handle" = ${actor}`;
|
|
11
8
|
}
|
|
12
9
|
};
|
|
13
|
-
exports.actorWhereClause = actorWhereClause;
|
|
14
10
|
// Applies to actor or record table
|
|
15
|
-
const notSoftDeletedClause = (alias) => {
|
|
16
|
-
return
|
|
11
|
+
export const notSoftDeletedClause = (alias) => {
|
|
12
|
+
return sql `${alias}."takedownRef" is null`;
|
|
17
13
|
};
|
|
18
|
-
|
|
19
|
-
const softDeleted = (actorOrRecord) => {
|
|
14
|
+
export const softDeleted = (actorOrRecord) => {
|
|
20
15
|
return actorOrRecord.takedownRef !== null;
|
|
21
16
|
};
|
|
22
|
-
|
|
23
|
-
exports.countAll = (0, kysely_1.sql) `count(*)`;
|
|
17
|
+
export const countAll = sql `count(*)`;
|
|
24
18
|
// For use with doUpdateSet()
|
|
25
|
-
const excluded = (db, col) => {
|
|
26
|
-
return
|
|
19
|
+
export const excluded = (db, col) => {
|
|
20
|
+
return sql `${db.dynamic.ref(`excluded.${col}`)}`;
|
|
27
21
|
};
|
|
28
|
-
|
|
29
|
-
exports.noMatch = (0, kysely_1.sql) `1 = 0`;
|
|
22
|
+
export const noMatch = sql `1 = 0`;
|
|
30
23
|
// Can be useful for large where-in clauses, to get the db to use a hash lookup on the list
|
|
31
|
-
const valuesList = (vals) => {
|
|
32
|
-
return
|
|
24
|
+
export const valuesList = (vals) => {
|
|
25
|
+
return sql `(values (${sql.join(vals, sql `), (`)}))`;
|
|
33
26
|
};
|
|
34
|
-
|
|
35
|
-
exports.dummyDialect = {
|
|
27
|
+
export const dummyDialect = {
|
|
36
28
|
createAdapter() {
|
|
37
|
-
return new
|
|
29
|
+
return new SqliteAdapter();
|
|
38
30
|
},
|
|
39
31
|
createDriver() {
|
|
40
|
-
return new
|
|
32
|
+
return new DummyDriver();
|
|
41
33
|
},
|
|
42
34
|
createIntrospector(db) {
|
|
43
|
-
return new
|
|
35
|
+
return new SqliteIntrospector(db);
|
|
44
36
|
},
|
|
45
37
|
createQueryCompiler() {
|
|
46
|
-
return new
|
|
38
|
+
return new SqliteQueryCompiler();
|
|
47
39
|
},
|
|
48
40
|
};
|
|
@@ -1,39 +1,24 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
const node_events_1 = __importDefault(require("node:events"));
|
|
11
|
-
const routes_1 = __importDefault(require("./routes"));
|
|
12
|
-
var subscription_1 = require("./subscription");
|
|
13
|
-
Object.defineProperty(exports, "RepoSubscription", { enumerable: true, get: function () { return subscription_1.RepoSubscription; } });
|
|
14
|
-
class DataPlaneServer {
|
|
1
|
+
import { IdResolver, MemoryCache } from '@atproto/identity';
|
|
2
|
+
import { expressConnectMiddleware } from '@connectrpc/connect-express';
|
|
3
|
+
import express from 'express';
|
|
4
|
+
import events from 'node:events';
|
|
5
|
+
import createRoutes from './routes';
|
|
6
|
+
export { RepoSubscription } from './subscription';
|
|
7
|
+
export class DataPlaneServer {
|
|
8
|
+
server;
|
|
9
|
+
idResolver;
|
|
15
10
|
constructor(server, idResolver) {
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
configurable: true,
|
|
19
|
-
writable: true,
|
|
20
|
-
value: server
|
|
21
|
-
});
|
|
22
|
-
Object.defineProperty(this, "idResolver", {
|
|
23
|
-
enumerable: true,
|
|
24
|
-
configurable: true,
|
|
25
|
-
writable: true,
|
|
26
|
-
value: idResolver
|
|
27
|
-
});
|
|
11
|
+
this.server = server;
|
|
12
|
+
this.idResolver = idResolver;
|
|
28
13
|
}
|
|
29
14
|
static async create(db, port, plcUrl) {
|
|
30
|
-
const app = (
|
|
31
|
-
const didCache = new
|
|
32
|
-
const idResolver = new
|
|
33
|
-
const routes = (
|
|
34
|
-
app.use(
|
|
15
|
+
const app = express();
|
|
16
|
+
const didCache = new MemoryCache();
|
|
17
|
+
const idResolver = new IdResolver({ plcUrl, didCache });
|
|
18
|
+
const routes = createRoutes(db, idResolver);
|
|
19
|
+
app.use(expressConnectMiddleware({ routes }));
|
|
35
20
|
const server = app.listen(port);
|
|
36
|
-
await
|
|
21
|
+
await events.once(server, 'listening');
|
|
37
22
|
return new DataPlaneServer(server, idResolver);
|
|
38
23
|
}
|
|
39
24
|
async destroy() {
|
|
@@ -49,4 +34,3 @@ class DataPlaneServer {
|
|
|
49
34
|
});
|
|
50
35
|
}
|
|
51
36
|
}
|
|
52
|
-
exports.DataPlaneServer = DataPlaneServer;
|
|
@@ -1,71 +1,22 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.IndexingService = void 0;
|
|
37
|
-
const api_1 = require("@atproto/api");
|
|
38
|
-
const common_1 = require("@atproto/common");
|
|
39
|
-
const identity_1 = require("@atproto/identity");
|
|
40
|
-
const lexicon_1 = require("@atproto/lexicon");
|
|
41
|
-
const repo_1 = require("@atproto/repo");
|
|
42
|
-
const syntax_1 = require("@atproto/syntax");
|
|
43
|
-
const kysely_1 = require("kysely");
|
|
44
|
-
const cid_1 = require("multiformats/cid");
|
|
45
|
-
const logger_1 = require("../../../logger");
|
|
46
|
-
const retry_1 = require("../../../util/retry");
|
|
47
|
-
const PlaylistIdea = __importStar(require("./plugins/playlist-idea"));
|
|
48
|
-
const Profile = __importStar(require("./plugins/profile"));
|
|
49
|
-
class IndexingService {
|
|
1
|
+
import { Agent, ComAtprotoSyncGetLatestCommit } from '@atproto/api';
|
|
2
|
+
import { DAY, HOUR } from '@atproto/common';
|
|
3
|
+
import { getPds } from '@atproto/identity';
|
|
4
|
+
import { ValidationError } from '@atproto/lexicon';
|
|
5
|
+
import { WriteOpAction, getAndParseRecord, readCarWithRoot, verifyRepo, } from '@atproto/repo';
|
|
6
|
+
import { AtUri } from '@atproto/syntax';
|
|
7
|
+
import { sql } from 'kysely';
|
|
8
|
+
import { CID } from 'multiformats/cid';
|
|
9
|
+
import { httpLogger } from '../../../logger';
|
|
10
|
+
import { retryXrpc } from '../../../util/retry';
|
|
11
|
+
import * as PlaylistIdea from './plugins/playlist-idea';
|
|
12
|
+
import * as Profile from './plugins/profile';
|
|
13
|
+
export class IndexingService {
|
|
14
|
+
db;
|
|
15
|
+
idResolver;
|
|
16
|
+
records;
|
|
50
17
|
constructor(db, idResolver) {
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
configurable: true,
|
|
54
|
-
writable: true,
|
|
55
|
-
value: db
|
|
56
|
-
});
|
|
57
|
-
Object.defineProperty(this, "idResolver", {
|
|
58
|
-
enumerable: true,
|
|
59
|
-
configurable: true,
|
|
60
|
-
writable: true,
|
|
61
|
-
value: idResolver
|
|
62
|
-
});
|
|
63
|
-
Object.defineProperty(this, "records", {
|
|
64
|
-
enumerable: true,
|
|
65
|
-
configurable: true,
|
|
66
|
-
writable: true,
|
|
67
|
-
value: void 0
|
|
68
|
-
});
|
|
18
|
+
this.db = db;
|
|
19
|
+
this.idResolver = idResolver;
|
|
69
20
|
this.records = {
|
|
70
21
|
playlistIdea: PlaylistIdea.makePlugin(this.db),
|
|
71
22
|
profile: Profile.makePlugin(this.db),
|
|
@@ -82,7 +33,7 @@ class IndexingService {
|
|
|
82
33
|
const indexer = indexingTx.findIndexerForCollection(uri.collection);
|
|
83
34
|
if (!indexer)
|
|
84
35
|
return;
|
|
85
|
-
if (action ===
|
|
36
|
+
if (action === WriteOpAction.Create) {
|
|
86
37
|
await indexer.insertRecord(uri, cid, obj, timestamp);
|
|
87
38
|
}
|
|
88
39
|
else {
|
|
@@ -140,10 +91,10 @@ class IndexingService {
|
|
|
140
91
|
this.db.assertNotTransaction();
|
|
141
92
|
const now = new Date().toISOString();
|
|
142
93
|
const { pds, signingKey } = await this.idResolver.did.resolveAtprotoData(did, true);
|
|
143
|
-
const agent = new
|
|
144
|
-
const { data: car } = await
|
|
145
|
-
const { root, blocks } = await
|
|
146
|
-
const verifiedRepo = await
|
|
94
|
+
const agent = new Agent(pds);
|
|
95
|
+
const { data: car } = await retryXrpc(() => agent.com.atproto.sync.getRepo({ did }));
|
|
96
|
+
const { root, blocks } = await readCarWithRoot(car);
|
|
97
|
+
const verifiedRepo = await verifyRepo(blocks, root, did, signingKey);
|
|
147
98
|
const currRecords = await this.getCurrentRecords(did);
|
|
148
99
|
const repoRecords = formatCheckout(did, verifiedRepo);
|
|
149
100
|
const diff = findDiffFromCheckout(currRecords, repoRecords);
|
|
@@ -154,16 +105,16 @@ class IndexingService {
|
|
|
154
105
|
await this.deleteRecord(uri);
|
|
155
106
|
}
|
|
156
107
|
else {
|
|
157
|
-
const parsed = await
|
|
158
|
-
await this.indexRecord(uri, cid, parsed.record, op.op === 'create' ?
|
|
108
|
+
const parsed = await getAndParseRecord(blocks, cid);
|
|
109
|
+
await this.indexRecord(uri, cid, parsed.record, op.op === 'create' ? WriteOpAction.Create : WriteOpAction.Update, now);
|
|
159
110
|
}
|
|
160
111
|
}
|
|
161
112
|
catch (err) {
|
|
162
|
-
if (err instanceof
|
|
163
|
-
|
|
113
|
+
if (err instanceof ValidationError) {
|
|
114
|
+
httpLogger.warn({ did, commit, uri: uri.toString(), cid: cid.toString() }, 'skipping indexing of invalid record');
|
|
164
115
|
}
|
|
165
116
|
else {
|
|
166
|
-
|
|
117
|
+
httpLogger.error({ err, did, commit, uri: uri.toString(), cid: cid.toString() }, 'skipping indexing due to error processing record');
|
|
167
118
|
}
|
|
168
119
|
}
|
|
169
120
|
}));
|
|
@@ -197,8 +148,8 @@ class IndexingService {
|
|
|
197
148
|
.onConflict((oc) => {
|
|
198
149
|
const excluded = (col) => ref(`excluded.${col}`);
|
|
199
150
|
return oc.column('did').doUpdateSet({
|
|
200
|
-
commitCid:
|
|
201
|
-
repoRev:
|
|
151
|
+
commitCid: sql `${excluded('commitCid')}`,
|
|
152
|
+
repoRev: sql `${excluded('repoRev')}`,
|
|
202
153
|
});
|
|
203
154
|
})
|
|
204
155
|
.execute();
|
|
@@ -211,8 +162,8 @@ class IndexingService {
|
|
|
211
162
|
.execute();
|
|
212
163
|
return res.reduce((acc, cur) => {
|
|
213
164
|
acc[cur.uri] = {
|
|
214
|
-
uri: new
|
|
215
|
-
cid:
|
|
165
|
+
uri: new AtUri(cur.uri),
|
|
166
|
+
cid: CID.parse(cur.cid),
|
|
216
167
|
};
|
|
217
168
|
return acc;
|
|
218
169
|
}, {});
|
|
@@ -231,16 +182,16 @@ class IndexingService {
|
|
|
231
182
|
}
|
|
232
183
|
async getActorIsHosted(did) {
|
|
233
184
|
const doc = await this.idResolver.did.resolve(did, true);
|
|
234
|
-
const pds = doc &&
|
|
185
|
+
const pds = doc && getPds(doc);
|
|
235
186
|
if (!pds)
|
|
236
187
|
return false;
|
|
237
|
-
const agent = new
|
|
188
|
+
const agent = new Agent(pds);
|
|
238
189
|
try {
|
|
239
|
-
await
|
|
190
|
+
await retryXrpc(() => agent.com.atproto.sync.getLatestCommit({ did }));
|
|
240
191
|
return true;
|
|
241
192
|
}
|
|
242
193
|
catch (err) {
|
|
243
|
-
if (err instanceof
|
|
194
|
+
if (err instanceof ComAtprotoSyncGetLatestCommit.RepoNotFoundError) {
|
|
244
195
|
return false;
|
|
245
196
|
}
|
|
246
197
|
return null;
|
|
@@ -272,7 +223,6 @@ class IndexingService {
|
|
|
272
223
|
.execute();
|
|
273
224
|
}
|
|
274
225
|
}
|
|
275
|
-
exports.IndexingService = IndexingService;
|
|
276
226
|
const findDiffFromCheckout = (curr, checkout) => {
|
|
277
227
|
const ops = [];
|
|
278
228
|
for (const uri of Object.keys(checkout)) {
|
|
@@ -299,7 +249,7 @@ const findDiffFromCheckout = (curr, checkout) => {
|
|
|
299
249
|
const formatCheckout = (did, verifiedRepo) => {
|
|
300
250
|
const records = {};
|
|
301
251
|
for (const create of verifiedRepo.creates) {
|
|
302
|
-
const uri =
|
|
252
|
+
const uri = AtUri.make(did, create.collection, create.rkey);
|
|
303
253
|
records[uri.toString()] = {
|
|
304
254
|
uri,
|
|
305
255
|
cid: create.cid,
|
|
@@ -312,10 +262,10 @@ const needsHandleReindex = (actor, timestamp) => {
|
|
|
312
262
|
return true;
|
|
313
263
|
const timeDiff = new Date(timestamp).getTime() - new Date(actor.indexed_at).getTime();
|
|
314
264
|
// revalidate daily
|
|
315
|
-
if (timeDiff >
|
|
265
|
+
if (timeDiff > DAY)
|
|
316
266
|
return true;
|
|
317
267
|
// revalidate more aggressively for invalidated handles
|
|
318
|
-
if (actor.handle === null && timeDiff >
|
|
268
|
+
if (actor.handle === null && timeDiff > HOUR)
|
|
319
269
|
return true;
|
|
320
270
|
return false;
|
|
321
271
|
};
|
|
@@ -1,42 +1,6 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
-
var ownKeys = function(o) {
|
|
20
|
-
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
-
var ar = [];
|
|
22
|
-
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
-
return ar;
|
|
24
|
-
};
|
|
25
|
-
return ownKeys(o);
|
|
26
|
-
};
|
|
27
|
-
return function (mod) {
|
|
28
|
-
if (mod && mod.__esModule) return mod;
|
|
29
|
-
var result = {};
|
|
30
|
-
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
-
__setModuleDefault(result, mod);
|
|
32
|
-
return result;
|
|
33
|
-
};
|
|
34
|
-
})();
|
|
35
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
-
exports.makePlugin = void 0;
|
|
37
|
-
const syntax_1 = require("@atproto/syntax");
|
|
38
|
-
const lex = __importStar(require("../../../../lexicons/lexicons"));
|
|
39
|
-
const processor_1 = require("../processor");
|
|
1
|
+
import { normalizeDatetimeAlways } from '@atproto/syntax';
|
|
2
|
+
import * as lex from '../../../../lexicons/lexicons';
|
|
3
|
+
import { RecordProcessor } from '../processor';
|
|
40
4
|
const lexId = lex.ids.ComClioplaylistsAlphaFeedPlaylistIdea;
|
|
41
5
|
const insertFn = async (db, uri, cid, obj, timestamp) => {
|
|
42
6
|
const masterPlaylist = obj.masterPlaylist;
|
|
@@ -98,7 +62,7 @@ const insertFn = async (db, uri, cid, obj, timestamp) => {
|
|
|
98
62
|
owner_did: obj.ownerDid,
|
|
99
63
|
master_playlist_id: masterPlaylistId,
|
|
100
64
|
recommended_playlist_id: recommendedPlaylistId,
|
|
101
|
-
created_at:
|
|
65
|
+
created_at: normalizeDatetimeAlways(obj.createdAt),
|
|
102
66
|
};
|
|
103
67
|
const [insertedPlaylistIdea] = await Promise.all([
|
|
104
68
|
db
|
|
@@ -151,13 +115,12 @@ const deleteFn = async (db, uri) => {
|
|
|
151
115
|
}
|
|
152
116
|
: null;
|
|
153
117
|
};
|
|
154
|
-
const makePlugin = (db) => {
|
|
155
|
-
return new
|
|
118
|
+
export const makePlugin = (db) => {
|
|
119
|
+
return new RecordProcessor(db, {
|
|
156
120
|
lexId,
|
|
157
121
|
insertFn,
|
|
158
122
|
findDuplicate,
|
|
159
123
|
deleteFn,
|
|
160
124
|
});
|
|
161
125
|
};
|
|
162
|
-
|
|
163
|
-
exports.default = exports.makePlugin;
|
|
126
|
+
export default makePlugin;
|