@atproto/bsky 0.0.81 → 0.0.82
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/dist/data-plane/client.d.ts.map +1 -1
- package/dist/data-plane/client.js +2 -1
- package/dist/data-plane/client.js.map +1 -1
- package/dist/data-plane/server/db/migrations/20240829T211238293Z-simplify-actor-sync.d.ts +4 -0
- package/dist/data-plane/server/db/migrations/20240829T211238293Z-simplify-actor-sync.d.ts.map +1 -0
- package/dist/data-plane/server/db/migrations/20240829T211238293Z-simplify-actor-sync.js +26 -0
- package/dist/data-plane/server/db/migrations/20240829T211238293Z-simplify-actor-sync.js.map +1 -0
- package/dist/data-plane/server/db/migrations/index.d.ts +1 -0
- package/dist/data-plane/server/db/migrations/index.d.ts.map +1 -1
- package/dist/data-plane/server/db/migrations/index.js +2 -1
- package/dist/data-plane/server/db/migrations/index.js.map +1 -1
- package/dist/data-plane/server/db/tables/actor-sync.d.ts +0 -3
- package/dist/data-plane/server/db/tables/actor-sync.d.ts.map +1 -1
- package/dist/data-plane/server/db/tables/actor-sync.js.map +1 -1
- package/dist/data-plane/server/indexing/index.d.ts +2 -7
- package/dist/data-plane/server/indexing/index.d.ts.map +1 -1
- package/dist/data-plane/server/indexing/index.js +4 -21
- package/dist/data-plane/server/indexing/index.js.map +1 -1
- package/dist/data-plane/server/subscription.d.ts +26 -0
- package/dist/data-plane/server/subscription.d.ts.map +1 -0
- package/dist/data-plane/server/subscription.js +115 -0
- package/dist/data-plane/server/subscription.js.map +1 -0
- package/package.json +4 -3
- package/src/data-plane/client.ts +4 -1
- package/src/data-plane/server/db/migrations/20240829T211238293Z-simplify-actor-sync.ts +23 -0
- package/src/data-plane/server/db/migrations/index.ts +1 -0
- package/src/data-plane/server/db/tables/actor-sync.ts +0 -3
- package/src/data-plane/server/indexing/index.ts +4 -25
- package/src/data-plane/server/subscription.ts +104 -0
- package/tests/data-plane/indexing.test.ts +1 -1
- package/tests/data-plane/{subscription/repo.test.ts → subscription.test.ts} +4 -9
- package/tests/views/actor-search.test.ts +1 -1
- package/dist/data-plane/server/subscription/index.d.ts +0 -33
- package/dist/data-plane/server/subscription/index.d.ts.map +0 -1
- package/dist/data-plane/server/subscription/index.js +0 -341
- package/dist/data-plane/server/subscription/index.js.map +0 -1
- package/dist/data-plane/server/subscription/util.d.ts +0 -65
- package/dist/data-plane/server/subscription/util.d.ts.map +0 -1
- package/dist/data-plane/server/subscription/util.js +0 -215
- package/dist/data-plane/server/subscription/util.js.map +0 -1
- package/src/data-plane/server/subscription/index.ts +0 -352
- package/src/data-plane/server/subscription/util.ts +0 -156
- package/tests/data-plane/subscription/util.test.ts +0 -185
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
import { Firehose, MemoryRunner } from '@atproto/sync'
|
|
2
|
+
import { IdResolver } from '@atproto/identity'
|
|
3
|
+
import { WriteOpAction } from '@atproto/repo'
|
|
4
|
+
import { subLogger as log } from '../../logger'
|
|
5
|
+
import { IndexingService } from './indexing'
|
|
6
|
+
import { Database } from './db'
|
|
7
|
+
import { BackgroundQueue } from './background'
|
|
8
|
+
|
|
9
|
+
export class RepoSubscription {
|
|
10
|
+
firehose: Firehose
|
|
11
|
+
runner: MemoryRunner
|
|
12
|
+
background: BackgroundQueue
|
|
13
|
+
indexingSvc: IndexingService
|
|
14
|
+
|
|
15
|
+
constructor(
|
|
16
|
+
public opts: { service: string; db: Database; idResolver: IdResolver },
|
|
17
|
+
) {
|
|
18
|
+
const { service, db, idResolver } = opts
|
|
19
|
+
this.background = new BackgroundQueue(db)
|
|
20
|
+
this.indexingSvc = new IndexingService(db, idResolver, this.background)
|
|
21
|
+
|
|
22
|
+
const { runner, firehose } = createFirehose({
|
|
23
|
+
idResolver,
|
|
24
|
+
service,
|
|
25
|
+
indexingSvc: this.indexingSvc,
|
|
26
|
+
})
|
|
27
|
+
this.runner = runner
|
|
28
|
+
this.firehose = firehose
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
start() {
|
|
32
|
+
this.firehose.start()
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
async restart() {
|
|
36
|
+
await this.destroy()
|
|
37
|
+
const { runner, firehose } = createFirehose({
|
|
38
|
+
idResolver: this.opts.idResolver,
|
|
39
|
+
service: this.opts.service,
|
|
40
|
+
indexingSvc: this.indexingSvc,
|
|
41
|
+
})
|
|
42
|
+
this.runner = runner
|
|
43
|
+
this.firehose = firehose
|
|
44
|
+
this.start()
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
async processAll() {
|
|
48
|
+
await this.runner.processAll()
|
|
49
|
+
await this.background.processAll()
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
async destroy() {
|
|
53
|
+
await this.firehose.destroy()
|
|
54
|
+
await this.runner.destroy()
|
|
55
|
+
await this.background.processAll()
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const createFirehose = (opts: {
|
|
60
|
+
idResolver: IdResolver
|
|
61
|
+
service: string
|
|
62
|
+
indexingSvc: IndexingService
|
|
63
|
+
}) => {
|
|
64
|
+
const { idResolver, service, indexingSvc } = opts
|
|
65
|
+
const runner = new MemoryRunner({ startCursor: 0 })
|
|
66
|
+
const firehose = new Firehose({
|
|
67
|
+
idResolver,
|
|
68
|
+
runner,
|
|
69
|
+
service,
|
|
70
|
+
unauthenticatedHandles: true, // indexing service handles these
|
|
71
|
+
unauthenticatedCommits: true, // @TODO there seems to be a very rare issue where the authenticator thinks a block is missing in deletion ops
|
|
72
|
+
onError: (err) => log.error({ err }, 'error in subscription'),
|
|
73
|
+
handleEvent: async (evt) => {
|
|
74
|
+
if (evt.event === 'identity') {
|
|
75
|
+
await indexingSvc.indexHandle(evt.did, evt.time, true)
|
|
76
|
+
} else if (evt.event === 'account') {
|
|
77
|
+
if (evt.active === false && evt.status === 'deleted') {
|
|
78
|
+
await indexingSvc.deleteActor(evt.did)
|
|
79
|
+
} else {
|
|
80
|
+
await indexingSvc.updateActorStatus(evt.did, evt.active, evt.status)
|
|
81
|
+
}
|
|
82
|
+
} else {
|
|
83
|
+
const indexFn =
|
|
84
|
+
evt.event === 'delete'
|
|
85
|
+
? indexingSvc.deleteRecord(evt.uri)
|
|
86
|
+
: indexingSvc.indexRecord(
|
|
87
|
+
evt.uri,
|
|
88
|
+
evt.cid,
|
|
89
|
+
evt.record,
|
|
90
|
+
evt.event === 'create'
|
|
91
|
+
? WriteOpAction.Create
|
|
92
|
+
: WriteOpAction.Update,
|
|
93
|
+
evt.time,
|
|
94
|
+
)
|
|
95
|
+
await Promise.all([
|
|
96
|
+
indexFn,
|
|
97
|
+
indexingSvc.setCommitLastSeen(evt.did, evt.commit, evt.rev),
|
|
98
|
+
indexingSvc.indexHandle(evt.did, evt.time),
|
|
99
|
+
])
|
|
100
|
+
}
|
|
101
|
+
},
|
|
102
|
+
})
|
|
103
|
+
return { firehose, runner }
|
|
104
|
+
}
|
|
@@ -429,7 +429,7 @@ describe('indexing', () => {
|
|
|
429
429
|
|
|
430
430
|
describe('indexRepo', () => {
|
|
431
431
|
beforeAll(async () => {
|
|
432
|
-
network.bsky.sub.
|
|
432
|
+
await network.bsky.sub.restart()
|
|
433
433
|
await basicSeed(sc, false)
|
|
434
434
|
await network.processAll()
|
|
435
435
|
await network.bsky.sub.destroy()
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { AtpAgent } from '@atproto/api'
|
|
2
2
|
import { cborDecode, cborEncode } from '@atproto/common'
|
|
3
|
-
import { DatabaseSchemaType } from '
|
|
3
|
+
import { DatabaseSchemaType } from '../../src/data-plane/server/db/database-schema'
|
|
4
4
|
import { SeedClient, TestNetwork, basicSeed } from '@atproto/dev-env'
|
|
5
5
|
import { PreparedWrite, sequencer } from '@atproto/pds'
|
|
6
6
|
import { CommitData } from '@atproto/repo'
|
|
7
|
-
import { ids } from '
|
|
8
|
-
import { forSnapshot } from '
|
|
7
|
+
import { ids } from '../../src/lexicon/lexicons'
|
|
8
|
+
import { forSnapshot } from '../_util'
|
|
9
9
|
|
|
10
10
|
type Database = TestNetwork['bsky']['db']
|
|
11
11
|
|
|
@@ -60,12 +60,7 @@ describe('sync', () => {
|
|
|
60
60
|
const originalTableDump = await getTableDump()
|
|
61
61
|
|
|
62
62
|
// Reprocess repos via sync subscription, on top of existing indices
|
|
63
|
-
await network.bsky.sub.
|
|
64
|
-
// Hard reset of state
|
|
65
|
-
network.bsky.sub.cursor = 0
|
|
66
|
-
network.bsky.sub.seenSeq = null
|
|
67
|
-
// Boot streams back up
|
|
68
|
-
network.bsky.sub.run()
|
|
63
|
+
await network.bsky.sub.restart()
|
|
69
64
|
await network.processAll()
|
|
70
65
|
|
|
71
66
|
// Permissive of indexedAt times changing
|
|
@@ -39,7 +39,7 @@ describe.skip('pds actor search views', () => {
|
|
|
39
39
|
.execute()
|
|
40
40
|
|
|
41
41
|
// Process remaining profiles
|
|
42
|
-
network.bsky.sub.
|
|
42
|
+
await network.bsky.sub.restart()
|
|
43
43
|
await network.processAll(50000)
|
|
44
44
|
headers = await network.serviceHeaders(
|
|
45
45
|
Object.values(sc.dids)[0],
|
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
/// <reference types="node" />
|
|
2
|
-
import { IdResolver } from '@atproto/identity';
|
|
3
|
-
import { IndexingService } from '../indexing';
|
|
4
|
-
import { Database } from '../db';
|
|
5
|
-
import { ConsecutiveList, PartitionedQueue } from './util';
|
|
6
|
-
import { BackgroundQueue } from '../background';
|
|
7
|
-
export declare class RepoSubscription {
|
|
8
|
-
private opts;
|
|
9
|
-
ac: AbortController;
|
|
10
|
-
running: Promise<void> | undefined;
|
|
11
|
-
cursor: number;
|
|
12
|
-
seenSeq: number | null;
|
|
13
|
-
repoQueue: PartitionedQueue;
|
|
14
|
-
consecutive: ConsecutiveList<number>;
|
|
15
|
-
background: BackgroundQueue;
|
|
16
|
-
indexingSvc: IndexingService;
|
|
17
|
-
constructor(opts: {
|
|
18
|
-
service: string;
|
|
19
|
-
db: Database;
|
|
20
|
-
idResolver: IdResolver;
|
|
21
|
-
background: BackgroundQueue;
|
|
22
|
-
});
|
|
23
|
-
run(): void;
|
|
24
|
-
private process;
|
|
25
|
-
private handleMessage;
|
|
26
|
-
private handleCommit;
|
|
27
|
-
private handleUpdateHandle;
|
|
28
|
-
private handleIdentityEvt;
|
|
29
|
-
private handleAccountEvt;
|
|
30
|
-
private getSubscription;
|
|
31
|
-
destroy(): Promise<void>;
|
|
32
|
-
}
|
|
33
|
-
//# sourceMappingURL=index.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../../../src/data-plane/server/subscription/index.ts"],"names":[],"mappings":";AAMA,OAAO,EAAE,UAAU,EAAE,MAAM,mBAAmB,CAAA;AAY9C,OAAO,EAAE,eAAe,EAAE,MAAM,aAAa,CAAA;AAC7C,OAAO,EAAE,QAAQ,EAAE,MAAM,OAAO,CAAA;AAChC,OAAO,EAEL,eAAe,EACf,gBAAgB,EAGjB,MAAM,QAAQ,CAAA;AACf,OAAO,EAAE,eAAe,EAAE,MAAM,eAAe,CAAA;AAE/C,qBAAa,gBAAgB;IAWzB,OAAO,CAAC,IAAI;IAVd,EAAE,kBAAwB;IAC1B,OAAO,EAAE,OAAO,CAAC,IAAI,CAAC,GAAG,SAAS,CAAA;IAClC,MAAM,SAAI;IACV,OAAO,EAAE,MAAM,GAAG,IAAI,CAAO;IAC7B,SAAS,mBAAkD;IAC3D,WAAW,0BAAgC;IAC3C,UAAU,EAAE,eAAe,CAAA;IAC3B,WAAW,EAAE,eAAe,CAAA;gBAGlB,IAAI,EAAE;QACZ,OAAO,EAAE,MAAM,CAAA;QACf,EAAE,EAAE,QAAQ,CAAA;QACZ,UAAU,EAAE,UAAU,CAAA;QACtB,UAAU,EAAE,eAAe,CAAA;KAC5B;IAUH,GAAG;YAgBW,OAAO;YAqBP,aAAa;YAqCb,YAAY;YAgEZ,kBAAkB;YAIlB,iBAAiB;YAIjB,gBAAgB;IAQ9B,OAAO,CAAC,eAAe;IAkCjB,OAAO;CAMd"}
|
|
@@ -1,341 +0,0 @@
|
|
|
1
|
-
"use strict";
|
|
2
|
-
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
-
if (k2 === undefined) k2 = k;
|
|
4
|
-
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
-
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
-
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
-
}
|
|
8
|
-
Object.defineProperty(o, k2, desc);
|
|
9
|
-
}) : (function(o, m, k, k2) {
|
|
10
|
-
if (k2 === undefined) k2 = k;
|
|
11
|
-
o[k2] = m[k];
|
|
12
|
-
}));
|
|
13
|
-
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
-
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
-
}) : function(o, v) {
|
|
16
|
-
o["default"] = v;
|
|
17
|
-
});
|
|
18
|
-
var __importStar = (this && this.__importStar) || function (mod) {
|
|
19
|
-
if (mod && mod.__esModule) return mod;
|
|
20
|
-
var result = {};
|
|
21
|
-
if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
|
|
22
|
-
__setModuleDefault(result, mod);
|
|
23
|
-
return result;
|
|
24
|
-
};
|
|
25
|
-
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
26
|
-
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
27
|
-
};
|
|
28
|
-
Object.defineProperty(exports, "__esModule", { value: true });
|
|
29
|
-
exports.RepoSubscription = void 0;
|
|
30
|
-
const node_assert_1 = __importDefault(require("node:assert"));
|
|
31
|
-
const syntax_1 = require("@atproto/syntax");
|
|
32
|
-
const xrpc_server_1 = require("@atproto/xrpc-server");
|
|
33
|
-
const common_1 = require("@atproto/common");
|
|
34
|
-
const lexicon_1 = require("@atproto/lexicon");
|
|
35
|
-
const repo_1 = require("@atproto/repo");
|
|
36
|
-
const lexicons_1 = require("../../../lexicon/lexicons");
|
|
37
|
-
const message = __importStar(require("../../../lexicon/types/com/atproto/sync/subscribeRepos"));
|
|
38
|
-
const logger_1 = require("../../../logger");
|
|
39
|
-
const indexing_1 = require("../indexing");
|
|
40
|
-
const util_1 = require("./util");
|
|
41
|
-
const background_1 = require("../background");
|
|
42
|
-
class RepoSubscription {
|
|
43
|
-
constructor(opts) {
|
|
44
|
-
Object.defineProperty(this, "opts", {
|
|
45
|
-
enumerable: true,
|
|
46
|
-
configurable: true,
|
|
47
|
-
writable: true,
|
|
48
|
-
value: opts
|
|
49
|
-
});
|
|
50
|
-
Object.defineProperty(this, "ac", {
|
|
51
|
-
enumerable: true,
|
|
52
|
-
configurable: true,
|
|
53
|
-
writable: true,
|
|
54
|
-
value: new AbortController()
|
|
55
|
-
});
|
|
56
|
-
Object.defineProperty(this, "running", {
|
|
57
|
-
enumerable: true,
|
|
58
|
-
configurable: true,
|
|
59
|
-
writable: true,
|
|
60
|
-
value: void 0
|
|
61
|
-
});
|
|
62
|
-
Object.defineProperty(this, "cursor", {
|
|
63
|
-
enumerable: true,
|
|
64
|
-
configurable: true,
|
|
65
|
-
writable: true,
|
|
66
|
-
value: 0
|
|
67
|
-
});
|
|
68
|
-
Object.defineProperty(this, "seenSeq", {
|
|
69
|
-
enumerable: true,
|
|
70
|
-
configurable: true,
|
|
71
|
-
writable: true,
|
|
72
|
-
value: null
|
|
73
|
-
});
|
|
74
|
-
Object.defineProperty(this, "repoQueue", {
|
|
75
|
-
enumerable: true,
|
|
76
|
-
configurable: true,
|
|
77
|
-
writable: true,
|
|
78
|
-
value: new util_1.PartitionedQueue({ concurrency: Infinity })
|
|
79
|
-
});
|
|
80
|
-
Object.defineProperty(this, "consecutive", {
|
|
81
|
-
enumerable: true,
|
|
82
|
-
configurable: true,
|
|
83
|
-
writable: true,
|
|
84
|
-
value: new util_1.ConsecutiveList()
|
|
85
|
-
});
|
|
86
|
-
Object.defineProperty(this, "background", {
|
|
87
|
-
enumerable: true,
|
|
88
|
-
configurable: true,
|
|
89
|
-
writable: true,
|
|
90
|
-
value: void 0
|
|
91
|
-
});
|
|
92
|
-
Object.defineProperty(this, "indexingSvc", {
|
|
93
|
-
enumerable: true,
|
|
94
|
-
configurable: true,
|
|
95
|
-
writable: true,
|
|
96
|
-
value: void 0
|
|
97
|
-
});
|
|
98
|
-
this.background = new background_1.BackgroundQueue(this.opts.db);
|
|
99
|
-
this.indexingSvc = new indexing_1.IndexingService(this.opts.db, this.opts.idResolver, this.background);
|
|
100
|
-
}
|
|
101
|
-
run() {
|
|
102
|
-
if (this.running)
|
|
103
|
-
return;
|
|
104
|
-
this.ac = new AbortController();
|
|
105
|
-
this.repoQueue = new util_1.PartitionedQueue({ concurrency: Infinity });
|
|
106
|
-
this.consecutive = new util_1.ConsecutiveList();
|
|
107
|
-
this.running = this.process()
|
|
108
|
-
.catch((err) => {
|
|
109
|
-
if (err.name !== 'AbortError') {
|
|
110
|
-
// allow this to cause an unhandled rejection, let deployment handle the crash.
|
|
111
|
-
logger_1.subLogger.error({ err }, 'subscription crashed');
|
|
112
|
-
throw err;
|
|
113
|
-
}
|
|
114
|
-
})
|
|
115
|
-
.finally(() => (this.running = undefined));
|
|
116
|
-
}
|
|
117
|
-
async process() {
|
|
118
|
-
const sub = this.getSubscription();
|
|
119
|
-
for await (const msg of sub) {
|
|
120
|
-
const details = getMessageDetails(msg);
|
|
121
|
-
if ('info' in details) {
|
|
122
|
-
// These messages are not sequenced, we just log them and carry on
|
|
123
|
-
logger_1.subLogger.warn({ provider: this.opts.service, message: (0, util_1.loggableMessage)(msg) }, `sub ${details.info ? 'info' : 'unknown'} message`);
|
|
124
|
-
continue;
|
|
125
|
-
}
|
|
126
|
-
const item = this.consecutive.push(details.seq);
|
|
127
|
-
this.repoQueue.add(details.repo, async () => {
|
|
128
|
-
await this.handleMessage(item, details);
|
|
129
|
-
});
|
|
130
|
-
this.seenSeq = details.seq;
|
|
131
|
-
await this.repoQueue.main.onEmpty(); // backpressure
|
|
132
|
-
}
|
|
133
|
-
}
|
|
134
|
-
async handleMessage(item, envelope) {
|
|
135
|
-
const msg = envelope.message;
|
|
136
|
-
try {
|
|
137
|
-
if (message.isCommit(msg)) {
|
|
138
|
-
await this.handleCommit(msg);
|
|
139
|
-
}
|
|
140
|
-
else if (message.isHandle(msg)) {
|
|
141
|
-
await this.handleUpdateHandle(msg);
|
|
142
|
-
}
|
|
143
|
-
else if (message.isIdentity(msg)) {
|
|
144
|
-
await this.handleIdentityEvt(msg);
|
|
145
|
-
}
|
|
146
|
-
else if (message.isAccount(msg)) {
|
|
147
|
-
await this.handleAccountEvt(msg);
|
|
148
|
-
}
|
|
149
|
-
else if (message.isTombstone(msg)) {
|
|
150
|
-
// Ignore tombstones
|
|
151
|
-
}
|
|
152
|
-
else if (message.isMigrate(msg)) {
|
|
153
|
-
// Ignore migrations
|
|
154
|
-
}
|
|
155
|
-
else {
|
|
156
|
-
const exhaustiveCheck = msg;
|
|
157
|
-
throw new Error(`Unhandled message type: ${exhaustiveCheck['$type']}`);
|
|
158
|
-
}
|
|
159
|
-
}
|
|
160
|
-
catch (err) {
|
|
161
|
-
// We log messages we can't process and move on:
|
|
162
|
-
// otherwise the cursor would get stuck on a poison message.
|
|
163
|
-
logger_1.subLogger.error({ err, message: (0, util_1.loggableMessage)(msg) }, 'indexer message processing error');
|
|
164
|
-
}
|
|
165
|
-
finally {
|
|
166
|
-
const latest = item.complete().at(-1);
|
|
167
|
-
if (latest !== undefined) {
|
|
168
|
-
this.cursor = latest;
|
|
169
|
-
}
|
|
170
|
-
}
|
|
171
|
-
}
|
|
172
|
-
async handleCommit(msg) {
|
|
173
|
-
const indexRecords = async () => {
|
|
174
|
-
const { root, rootCid, ops } = await getOps(msg);
|
|
175
|
-
if (msg.tooBig) {
|
|
176
|
-
await this.indexingSvc.indexRepo(msg.repo, rootCid.toString());
|
|
177
|
-
await this.indexingSvc.setCommitLastSeen(root, msg);
|
|
178
|
-
return;
|
|
179
|
-
}
|
|
180
|
-
if (msg.rebase) {
|
|
181
|
-
const needsReindex = await this.indexingSvc.checkCommitNeedsIndexing(root);
|
|
182
|
-
if (needsReindex) {
|
|
183
|
-
await this.indexingSvc.indexRepo(msg.repo, rootCid.toString());
|
|
184
|
-
}
|
|
185
|
-
await this.indexingSvc.setCommitLastSeen(root, msg);
|
|
186
|
-
return;
|
|
187
|
-
}
|
|
188
|
-
for (const op of ops) {
|
|
189
|
-
if (op.action === repo_1.WriteOpAction.Delete) {
|
|
190
|
-
await this.indexingSvc.deleteRecord(op.uri);
|
|
191
|
-
}
|
|
192
|
-
else {
|
|
193
|
-
try {
|
|
194
|
-
await this.indexingSvc.indexRecord(op.uri, op.cid, op.record, op.action, // create or update
|
|
195
|
-
msg.time);
|
|
196
|
-
}
|
|
197
|
-
catch (err) {
|
|
198
|
-
if (err instanceof lexicon_1.ValidationError) {
|
|
199
|
-
logger_1.subLogger.warn({
|
|
200
|
-
did: msg.repo,
|
|
201
|
-
commit: msg.commit.toString(),
|
|
202
|
-
uri: op.uri.toString(),
|
|
203
|
-
cid: op.cid.toString(),
|
|
204
|
-
}, 'skipping indexing of invalid record');
|
|
205
|
-
}
|
|
206
|
-
else {
|
|
207
|
-
logger_1.subLogger.error({
|
|
208
|
-
err,
|
|
209
|
-
did: msg.repo,
|
|
210
|
-
commit: msg.commit.toString(),
|
|
211
|
-
uri: op.uri.toString(),
|
|
212
|
-
cid: op.cid.toString(),
|
|
213
|
-
}, 'skipping indexing due to error processing record');
|
|
214
|
-
}
|
|
215
|
-
}
|
|
216
|
-
}
|
|
217
|
-
}
|
|
218
|
-
await this.indexingSvc.setCommitLastSeen(root, msg);
|
|
219
|
-
};
|
|
220
|
-
const results = await Promise.allSettled([
|
|
221
|
-
indexRecords(),
|
|
222
|
-
this.indexingSvc.indexHandle(msg.repo, msg.time),
|
|
223
|
-
]);
|
|
224
|
-
(0, common_1.handleAllSettledErrors)(results);
|
|
225
|
-
}
|
|
226
|
-
async handleUpdateHandle(msg) {
|
|
227
|
-
await this.indexingSvc.indexHandle(msg.did, msg.time, true);
|
|
228
|
-
}
|
|
229
|
-
async handleIdentityEvt(msg) {
|
|
230
|
-
await this.indexingSvc.indexHandle(msg.did, msg.time, true);
|
|
231
|
-
}
|
|
232
|
-
async handleAccountEvt(msg) {
|
|
233
|
-
if (msg.active === false && msg.status === 'deleted') {
|
|
234
|
-
await this.indexingSvc.deleteActor(msg.did);
|
|
235
|
-
}
|
|
236
|
-
else {
|
|
237
|
-
await this.indexingSvc.updateActorStatus(msg.did, msg.active, msg.status);
|
|
238
|
-
}
|
|
239
|
-
}
|
|
240
|
-
getSubscription() {
|
|
241
|
-
return new xrpc_server_1.Subscription({
|
|
242
|
-
service: this.opts.service,
|
|
243
|
-
method: lexicons_1.ids.ComAtprotoSyncSubscribeRepos,
|
|
244
|
-
signal: this.ac.signal,
|
|
245
|
-
getParams: async () => {
|
|
246
|
-
return { cursor: this.cursor };
|
|
247
|
-
},
|
|
248
|
-
onReconnectError: (err, reconnects, initial) => {
|
|
249
|
-
logger_1.subLogger.warn({ err, reconnects, initial }, 'sub reconnect');
|
|
250
|
-
},
|
|
251
|
-
validate: (value) => {
|
|
252
|
-
try {
|
|
253
|
-
return lexicons_1.lexicons.assertValidXrpcMessage(lexicons_1.ids.ComAtprotoSyncSubscribeRepos, value);
|
|
254
|
-
}
|
|
255
|
-
catch (err) {
|
|
256
|
-
logger_1.subLogger.warn({
|
|
257
|
-
err,
|
|
258
|
-
seq: ifNumber(value?.['seq']),
|
|
259
|
-
repo: ifString(value?.['repo']),
|
|
260
|
-
commit: ifString(value?.['commit']?.toString()),
|
|
261
|
-
time: ifString(value?.['time']),
|
|
262
|
-
provider: this.opts.service,
|
|
263
|
-
}, 'ingester sub skipped invalid message');
|
|
264
|
-
}
|
|
265
|
-
},
|
|
266
|
-
});
|
|
267
|
-
}
|
|
268
|
-
async destroy() {
|
|
269
|
-
this.ac.abort();
|
|
270
|
-
await this.running;
|
|
271
|
-
await this.repoQueue.destroy();
|
|
272
|
-
await this.background.processAll();
|
|
273
|
-
}
|
|
274
|
-
}
|
|
275
|
-
exports.RepoSubscription = RepoSubscription;
|
|
276
|
-
function ifString(val) {
|
|
277
|
-
return typeof val === 'string' ? val : undefined;
|
|
278
|
-
}
|
|
279
|
-
function ifNumber(val) {
|
|
280
|
-
return typeof val === 'number' ? val : undefined;
|
|
281
|
-
}
|
|
282
|
-
function getMessageDetails(msg) {
|
|
283
|
-
if (message.isCommit(msg)) {
|
|
284
|
-
return { seq: msg.seq, repo: msg.repo, message: msg };
|
|
285
|
-
}
|
|
286
|
-
else if (message.isHandle(msg)) {
|
|
287
|
-
return { seq: msg.seq, repo: msg.did, message: msg };
|
|
288
|
-
}
|
|
289
|
-
else if (message.isIdentity(msg)) {
|
|
290
|
-
return { seq: msg.seq, repo: msg.did, message: msg };
|
|
291
|
-
}
|
|
292
|
-
else if (message.isAccount(msg)) {
|
|
293
|
-
return { seq: msg.seq, repo: msg.did, message: msg };
|
|
294
|
-
}
|
|
295
|
-
else if (message.isMigrate(msg)) {
|
|
296
|
-
return { seq: msg.seq, repo: msg.did, message: msg };
|
|
297
|
-
}
|
|
298
|
-
else if (message.isTombstone(msg)) {
|
|
299
|
-
return { seq: msg.seq, repo: msg.did, message: msg };
|
|
300
|
-
}
|
|
301
|
-
else if (message.isInfo(msg)) {
|
|
302
|
-
return { info: msg };
|
|
303
|
-
}
|
|
304
|
-
return { info: null };
|
|
305
|
-
}
|
|
306
|
-
async function getOps(msg) {
|
|
307
|
-
const car = await (0, repo_1.readCarWithRoot)(msg.blocks);
|
|
308
|
-
const rootBytes = car.blocks.get(car.root);
|
|
309
|
-
(0, node_assert_1.default)(rootBytes, 'Missing commit block in car slice');
|
|
310
|
-
const root = repo_1.def.commit.schema.parse((0, common_1.cborDecode)(rootBytes));
|
|
311
|
-
const ops = msg.ops.map((op) => {
|
|
312
|
-
const [collection, rkey] = op.path.split('/');
|
|
313
|
-
(0, node_assert_1.default)(collection && rkey);
|
|
314
|
-
if (op.action === repo_1.WriteOpAction.Create ||
|
|
315
|
-
op.action === repo_1.WriteOpAction.Update) {
|
|
316
|
-
(0, node_assert_1.default)(op.cid);
|
|
317
|
-
const record = car.blocks.get(op.cid);
|
|
318
|
-
(0, node_assert_1.default)(record);
|
|
319
|
-
return {
|
|
320
|
-
action: op.action === repo_1.WriteOpAction.Create
|
|
321
|
-
? repo_1.WriteOpAction.Create
|
|
322
|
-
: repo_1.WriteOpAction.Update,
|
|
323
|
-
cid: op.cid,
|
|
324
|
-
record: (0, repo_1.cborToLexRecord)(record),
|
|
325
|
-
blobs: [],
|
|
326
|
-
uri: syntax_1.AtUri.make(msg.repo, collection, rkey),
|
|
327
|
-
};
|
|
328
|
-
}
|
|
329
|
-
else if (op.action === repo_1.WriteOpAction.Delete) {
|
|
330
|
-
return {
|
|
331
|
-
action: repo_1.WriteOpAction.Delete,
|
|
332
|
-
uri: syntax_1.AtUri.make(msg.repo, collection, rkey),
|
|
333
|
-
};
|
|
334
|
-
}
|
|
335
|
-
else {
|
|
336
|
-
throw new Error(`Unknown repo op action: ${op.action}`);
|
|
337
|
-
}
|
|
338
|
-
});
|
|
339
|
-
return { root, rootCid: car.root, ops };
|
|
340
|
-
}
|
|
341
|
-
//# sourceMappingURL=index.js.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../src/data-plane/server/subscription/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAA,8DAAgC;AAEhC,4CAAuC;AACvC,sDAAmD;AACnD,4CAAoE;AACpE,8CAAkD;AAElD,wCAMsB;AACtB,wDAAyD;AAEzD,gGAAiF;AACjF,4CAAkD;AAClD,0CAA6C;AAE7C,iCAMe;AACf,8CAA+C;AAE/C,MAAa,gBAAgB;IAU3B,YACU,IAKP;QALD;;;;mBAAQ,IAAI;WAKX;QAfH;;;;mBAAK,IAAI,eAAe,EAAE;WAAA;QAC1B;;;;;WAAkC;QAClC;;;;mBAAS,CAAC;WAAA;QACV;;;;mBAAyB,IAAI;WAAA;QAC7B;;;;mBAAY,IAAI,uBAAgB,CAAC,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;WAAA;QAC3D;;;;mBAAc,IAAI,sBAAe,EAAU;WAAA;QAC3C;;;;;WAA2B;QAC3B;;;;;WAA4B;QAU1B,IAAI,CAAC,UAAU,GAAG,IAAI,4BAAe,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,CAAA;QACnD,IAAI,CAAC,WAAW,GAAG,IAAI,0BAAe,CACpC,IAAI,CAAC,IAAI,CAAC,EAAE,EACZ,IAAI,CAAC,IAAI,CAAC,UAAU,EACpB,IAAI,CAAC,UAAU,CAChB,CAAA;IACH,CAAC;IAED,GAAG;QACD,IAAI,IAAI,CAAC,OAAO;YAAE,OAAM;QACxB,IAAI,CAAC,EAAE,GAAG,IAAI,eAAe,EAAE,CAAA;QAC/B,IAAI,CAAC,SAAS,GAAG,IAAI,uBAAgB,CAAC,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC,CAAA;QAChE,IAAI,CAAC,WAAW,GAAG,IAAI,sBAAe,EAAU,CAAA;QAChD,IAAI,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,EAAE;aAC1B,KAAK,CAAC,CAAC,GAAG,EAAE,EAAE;YACb,IAAI,GAAG,CAAC,IAAI,KAAK,YAAY,EAAE,CAAC;gBAC9B,+EAA+E;gBAC/E,kBAAG,CAAC,KAAK,CAAC,EAAE,GAAG,EAAE,EAAE,sBAAsB,CAAC,CAAA;gBAC1C,MAAM,GAAG,CAAA;YACX,CAAC;QACH,CAAC,CAAC;aACD,OAAO,CAAC,GAAG,EAAE,CAAC,CAAC,IAAI,CAAC,OAAO,GAAG,SAAS,CAAC,CAAC,CAAA;IAC9C,CAAC;IAEO,KAAK,CAAC,OAAO;QACnB,MAAM,GAAG,GAAG,IAAI,CAAC,eAAe,EAAE,CAAA;QAClC,IAAI,KAAK,EAAE,MAAM,GAAG,IAAI,GAAG,EAAE,CAAC;YAC5B,MAAM,OAAO,GAAG,iBAAiB,CAAC,GAAG,CAAC,CAAA;YACtC,IAAI,MAAM,IAAI,OAAO,EAAE,CAAC;gBACtB,kEAAkE;gBAClE,kBAAG,CAAC,IAAI,CACN,EAAE,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO,EAAE,OAAO,EAAE,IAAA,sBAAe,EAAC,GAAG,CAAC,EAAE,EAC9D,OAAO,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,UAAU,CACnD,CAAA;gBACD,SAAQ;YACV,CAAC;YACD,MAAM,IAAI,GAAG,IAAI,CAAC,WAAW,CAAC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,CAAA;YAC/C,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,IAAI,EAAE,KAAK,IAAI,EAAE;gBAC1C,MAAM,IAAI,CAAC,aAAa,CAAC,IAAI,EAAE,OAAO,CAAC,CAAA;YACzC,CAAC,CAAC,CAAA;YACF,IAAI,CAAC,OAAO,GAAG,OAAO,CAAC,GAAG,CAAA;YAC1B,MAAM,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,OAAO,EAAE,CAAA,CAAC,eAAe;QACrD,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,aAAa,CACzB,IAA6B,EAC7B,QAAkB;QAElB,MAAM,GAAG,GAAG,QAAQ,CAAC,OAAO,CAAA;QAC5B,IAAI,CAAC;YACH,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBAC1B,MAAM,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,CAAA;YAC9B,CAAC;iBAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;gBACjC,MAAM,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,CAAA;YACpC,CAAC;iBAAM,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;gBACnC,MAAM,IAAI,CAAC,iBAAiB,CAAC,GAAG,CAAC,CAAA;YACnC,CAAC;iBAAM,IAAI,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClC,MAAM,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,CAAA;YAClC,CAAC;iBAAM,IAAI,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;gBACpC,oBAAoB;YACtB,CAAC;iBAAM,IAAI,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;gBAClC,oBAAoB;YACtB,CAAC;iBAAM,CAAC;gBACN,MAAM,eAAe,GAAU,GAAG,CAAA;gBAClC,MAAM,IAAI,KAAK,CAAC,2BAA2B,eAAe,CAAC,OAAO,CAAC,EAAE,CAAC,CAAA;YACxE,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,gDAAgD;YAChD,4DAA4D;YAC5D,kBAAG,CAAC,KAAK,CACP,EAAE,GAAG,EAAE,OAAO,EAAE,IAAA,sBAAe,EAAC,GAAG,CAAC,EAAE,EACtC,kCAAkC,CACnC,CAAA;QACH,CAAC;gBAAS,CAAC;YACT,MAAM,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,CAAA;YACrC,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,IAAI,CAAC,MAAM,GAAG,MAAM,CAAA;YACtB,CAAC;QACH,CAAC;IACH,CAAC;IAEO,KAAK,CAAC,YAAY,CAAC,GAAmB;QAC5C,MAAM,YAAY,GAAG,KAAK,IAAI,EAAE;YAC9B,MAAM,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,GAAG,MAAM,MAAM,CAAC,GAAG,CAAC,CAAA;YAChD,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;gBACf,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAA;gBAC9D,MAAM,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;gBACnD,OAAM;YACR,CAAC;YACD,IAAI,GAAG,CAAC,MAAM,EAAE,CAAC;gBACf,MAAM,YAAY,GAChB,MAAM,IAAI,CAAC,WAAW,CAAC,wBAAwB,CAAC,IAAI,CAAC,CAAA;gBACvD,IAAI,YAAY,EAAE,CAAC;oBACjB,MAAM,IAAI,CAAC,WAAW,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,QAAQ,EAAE,CAAC,CAAA;gBAChE,CAAC;gBACD,MAAM,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;gBACnD,OAAM;YACR,CAAC;YACD,KAAK,MAAM,EAAE,IAAI,GAAG,EAAE,CAAC;gBACrB,IAAI,EAAE,CAAC,MAAM,KAAK,oBAAa,CAAC,MAAM,EAAE,CAAC;oBACvC,MAAM,IAAI,CAAC,WAAW,CAAC,YAAY,CAAC,EAAE,CAAC,GAAG,CAAC,CAAA;gBAC7C,CAAC;qBAAM,CAAC;oBACN,IAAI,CAAC;wBACH,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAChC,EAAE,CAAC,GAAG,EACN,EAAE,CAAC,GAAG,EACN,EAAE,CAAC,MAAM,EACT,EAAE,CAAC,MAAM,EAAE,mBAAmB;wBAC9B,GAAG,CAAC,IAAI,CACT,CAAA;oBACH,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,IAAI,GAAG,YAAY,yBAAe,EAAE,CAAC;4BACnC,kBAAG,CAAC,IAAI,CACN;gCACE,GAAG,EAAE,GAAG,CAAC,IAAI;gCACb,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE;gCAC7B,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE;gCACtB,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE;6BACvB,EACD,qCAAqC,CACtC,CAAA;wBACH,CAAC;6BAAM,CAAC;4BACN,kBAAG,CAAC,KAAK,CACP;gCACE,GAAG;gCACH,GAAG,EAAE,GAAG,CAAC,IAAI;gCACb,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,QAAQ,EAAE;gCAC7B,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE;gCACtB,GAAG,EAAE,EAAE,CAAC,GAAG,CAAC,QAAQ,EAAE;6BACvB,EACD,kDAAkD,CACnD,CAAA;wBACH,CAAC;oBACH,CAAC;gBACH,CAAC;YACH,CAAC;YACD,MAAM,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,IAAI,EAAE,GAAG,CAAC,CAAA;QACrD,CAAC,CAAA;QACD,MAAM,OAAO,GAAG,MAAM,OAAO,CAAC,UAAU,CAAC;YACvC,YAAY,EAAE;YACd,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,IAAI,EAAE,GAAG,CAAC,IAAI,CAAC;SACjD,CAAC,CAAA;QACF,IAAA,+BAAsB,EAAC,OAAO,CAAC,CAAA;IACjC,CAAC;IAEO,KAAK,CAAC,kBAAkB,CAAC,GAAmB;QAClD,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;IAC7D,CAAC;IAEO,KAAK,CAAC,iBAAiB,CAAC,GAAqB;QACnD,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,IAAI,EAAE,IAAI,CAAC,CAAA;IAC7D,CAAC;IAEO,KAAK,CAAC,gBAAgB,CAAC,GAAoB;QACjD,IAAI,GAAG,CAAC,MAAM,KAAK,KAAK,IAAI,GAAG,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACrD,MAAM,IAAI,CAAC,WAAW,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,CAAA;QAC7C,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,CAAC,WAAW,CAAC,iBAAiB,CAAC,GAAG,CAAC,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,CAAA;QAC3E,CAAC;IACH,CAAC;IAEO,eAAe;QACrB,OAAO,IAAI,0BAAY,CAAC;YACtB,OAAO,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO;YAC1B,MAAM,EAAE,cAAG,CAAC,4BAA4B;YACxC,MAAM,EAAE,IAAI,CAAC,EAAE,CAAC,MAAM;YACtB,SAAS,EAAE,KAAK,IAAI,EAAE;gBACpB,OAAO,EAAE,MAAM,EAAE,IAAI,CAAC,MAAM,EAAE,CAAA;YAChC,CAAC;YACD,gBAAgB,EAAE,CAAC,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE;gBAC7C,kBAAG,CAAC,IAAI,CAAC,EAAE,GAAG,EAAE,UAAU,EAAE,OAAO,EAAE,EAAE,eAAe,CAAC,CAAA;YACzD,CAAC;YACD,QAAQ,EAAE,CAAC,KAAK,EAAE,EAAE;gBAClB,IAAI,CAAC;oBACH,OAAO,mBAAQ,CAAC,sBAAsB,CACpC,cAAG,CAAC,4BAA4B,EAChC,KAAK,CACN,CAAA;gBACH,CAAC;gBAAC,OAAO,GAAG,EAAE,CAAC;oBACb,kBAAG,CAAC,IAAI,CACN;wBACE,GAAG;wBACH,GAAG,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,KAAK,CAAC,CAAC;wBAC7B,IAAI,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,CAAC;wBAC/B,MAAM,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,QAAQ,CAAC,EAAE,QAAQ,EAAE,CAAC;wBAC/C,IAAI,EAAE,QAAQ,CAAC,KAAK,EAAE,CAAC,MAAM,CAAC,CAAC;wBAC/B,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,OAAO;qBAC5B,EACD,sCAAsC,CACvC,CAAA;gBACH,CAAC;YACH,CAAC;SACF,CAAC,CAAA;IACJ,CAAC;IAED,KAAK,CAAC,OAAO;QACX,IAAI,CAAC,EAAE,CAAC,KAAK,EAAE,CAAA;QACf,MAAM,IAAI,CAAC,OAAO,CAAA;QAClB,MAAM,IAAI,CAAC,SAAS,CAAC,OAAO,EAAE,CAAA;QAC9B,MAAM,IAAI,CAAC,UAAU,CAAC,UAAU,EAAE,CAAA;IACpC,CAAC;CACF;AA5ND,4CA4NC;AAOD,SAAS,QAAQ,CAAC,GAAY;IAC5B,OAAO,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAA;AAClD,CAAC;AAED,SAAS,QAAQ,CAAC,GAAY;IAC5B,OAAO,OAAO,GAAG,KAAK,QAAQ,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAA;AAClD,CAAC;AAED,SAAS,iBAAiB,CAAC,GAAY;IAOrC,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,IAAI,EAAE,OAAO,EAAE,GAAG,EAAE,CAAA;IACvD,CAAC;SAAM,IAAI,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QACjC,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAA;IACtD,CAAC;SAAM,IAAI,OAAO,CAAC,UAAU,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAA;IACtD,CAAC;SAAM,IAAI,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAA;IACtD,CAAC;SAAM,IAAI,OAAO,CAAC,SAAS,CAAC,GAAG,CAAC,EAAE,CAAC;QAClC,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAA;IACtD,CAAC;SAAM,IAAI,OAAO,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;QACpC,OAAO,EAAE,GAAG,EAAE,GAAG,CAAC,GAAG,EAAE,IAAI,EAAE,GAAG,CAAC,GAAG,EAAE,OAAO,EAAE,GAAG,EAAE,CAAA;IACtD,CAAC;SAAM,IAAI,OAAO,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/B,OAAO,EAAE,IAAI,EAAE,GAAG,EAAE,CAAA;IACtB,CAAC;IACD,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAA;AACvB,CAAC;AAED,KAAK,UAAU,MAAM,CACnB,GAAmB;IAEnB,MAAM,GAAG,GAAG,MAAM,IAAA,sBAAe,EAAC,GAAG,CAAC,MAAoB,CAAC,CAAA;IAC3D,MAAM,SAAS,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,GAAG,CAAC,IAAI,CAAC,CAAA;IAC1C,IAAA,qBAAM,EAAC,SAAS,EAAE,mCAAmC,CAAC,CAAA;IAEtD,MAAM,IAAI,GAAG,UAAG,CAAC,MAAM,CAAC,MAAM,CAAC,KAAK,CAAC,IAAA,mBAAU,EAAC,SAAS,CAAC,CAAC,CAAA;IAC3D,MAAM,GAAG,GAAoB,GAAG,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,EAAE,EAAE,EAAE;QAC9C,MAAM,CAAC,UAAU,EAAE,IAAI,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,KAAK,CAAC,GAAG,CAAC,CAAA;QAC7C,IAAA,qBAAM,EAAC,UAAU,IAAI,IAAI,CAAC,CAAA;QAC1B,IACE,EAAE,CAAC,MAAM,KAAK,oBAAa,CAAC,MAAM;YAClC,EAAE,CAAC,MAAM,KAAK,oBAAa,CAAC,MAAM,EAClC,CAAC;YACD,IAAA,qBAAM,EAAC,EAAE,CAAC,GAAG,CAAC,CAAA;YACd,MAAM,MAAM,GAAG,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,CAAC,CAAA;YACrC,IAAA,qBAAM,EAAC,MAAM,CAAC,CAAA;YACd,OAAO;gBACL,MAAM,EACJ,EAAE,CAAC,MAAM,KAAK,oBAAa,CAAC,MAAM;oBAChC,CAAC,CAAC,oBAAa,CAAC,MAAM;oBACtB,CAAC,CAAC,oBAAa,CAAC,MAAM;gBAC1B,GAAG,EAAE,EAAE,CAAC,GAAG;gBACX,MAAM,EAAE,IAAA,sBAAe,EAAC,MAAM,CAAC;gBAC/B,KAAK,EAAE,EAAE;gBACT,GAAG,EAAE,cAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC;aAC5C,CAAA;QACH,CAAC;aAAM,IAAI,EAAE,CAAC,MAAM,KAAK,oBAAa,CAAC,MAAM,EAAE,CAAC;YAC9C,OAAO;gBACL,MAAM,EAAE,oBAAa,CAAC,MAAM;gBAC5B,GAAG,EAAE,cAAK,CAAC,IAAI,CAAC,GAAG,CAAC,IAAI,EAAE,UAAU,EAAE,IAAI,CAAC;aAC5C,CAAA;QACH,CAAC;aAAM,CAAC;YACN,MAAM,IAAI,KAAK,CAAC,2BAA2B,EAAE,CAAC,MAAM,EAAE,CAAC,CAAA;QACzD,CAAC;IACH,CAAC,CAAC,CAAA;IAEF,OAAO,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,CAAC,IAAI,EAAE,GAAG,EAAE,CAAA;AACzC,CAAC"}
|
|
@@ -1,65 +0,0 @@
|
|
|
1
|
-
import PQueue from 'p-queue';
|
|
2
|
-
import { OutputSchema as RepoMessage } from '../../../lexicon/types/com/atproto/sync/subscribeRepos';
|
|
3
|
-
import * as message from '../../../lexicon/types/com/atproto/sync/subscribeRepos';
|
|
4
|
-
export declare class PartitionedQueue {
|
|
5
|
-
main: PQueue;
|
|
6
|
-
partitions: Map<string, PQueue<import("p-queue/dist/priority-queue").default, import("p-queue").DefaultAddOptions>>;
|
|
7
|
-
constructor(opts: {
|
|
8
|
-
concurrency: number;
|
|
9
|
-
});
|
|
10
|
-
add(partitionId: string, task: () => Promise<void>): Promise<void>;
|
|
11
|
-
destroy(): Promise<void>;
|
|
12
|
-
private getPartition;
|
|
13
|
-
}
|
|
14
|
-
export declare class LatestQueue {
|
|
15
|
-
queue: PQueue<import("p-queue/dist/priority-queue").default, import("p-queue").DefaultAddOptions>;
|
|
16
|
-
add(task: () => Promise<void>): Promise<void>;
|
|
17
|
-
destroy(): Promise<void>;
|
|
18
|
-
}
|
|
19
|
-
/**
|
|
20
|
-
* Add items to a list, and mark those items as
|
|
21
|
-
* completed. Upon item completion, get list of consecutive
|
|
22
|
-
* items completed at the head of the list. Example:
|
|
23
|
-
*
|
|
24
|
-
* const consecutive = new ConsecutiveList<number>()
|
|
25
|
-
* const item1 = consecutive.push(1)
|
|
26
|
-
* const item2 = consecutive.push(2)
|
|
27
|
-
* const item3 = consecutive.push(3)
|
|
28
|
-
* item2.complete() // []
|
|
29
|
-
* item1.complete() // [1, 2]
|
|
30
|
-
* item3.complete() // [3]
|
|
31
|
-
*
|
|
32
|
-
*/
|
|
33
|
-
export declare class ConsecutiveList<T> {
|
|
34
|
-
list: ConsecutiveItem<T>[];
|
|
35
|
-
push(value: T): ConsecutiveItem<T>;
|
|
36
|
-
complete(): T[];
|
|
37
|
-
}
|
|
38
|
-
export declare class ConsecutiveItem<T> {
|
|
39
|
-
private consecutive;
|
|
40
|
-
value: T;
|
|
41
|
-
isComplete: boolean;
|
|
42
|
-
constructor(consecutive: ConsecutiveList<T>, value: T);
|
|
43
|
-
complete(): T[];
|
|
44
|
-
}
|
|
45
|
-
export declare class PerfectMap<K, V> extends Map<K, V> {
|
|
46
|
-
get(key: K): V;
|
|
47
|
-
}
|
|
48
|
-
export type ProcessableMessage = message.Commit | message.Handle | message.Identity | message.Migrate | message.Tombstone;
|
|
49
|
-
export declare function loggableMessage(msg: RepoMessage): message.Tombstone | message.Info | {
|
|
50
|
-
[k: string]: unknown;
|
|
51
|
-
$type: string;
|
|
52
|
-
} | {
|
|
53
|
-
$type: unknown;
|
|
54
|
-
seq: number;
|
|
55
|
-
rebase: boolean;
|
|
56
|
-
prev: string | undefined;
|
|
57
|
-
repo: string;
|
|
58
|
-
commit: string;
|
|
59
|
-
time: string;
|
|
60
|
-
tooBig: boolean;
|
|
61
|
-
hasBlobs: boolean;
|
|
62
|
-
};
|
|
63
|
-
export declare function jitter(maxMs: any): number;
|
|
64
|
-
export declare function strToInt(str: string): number;
|
|
65
|
-
//# sourceMappingURL=util.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"util.d.ts","sourceRoot":"","sources":["../../../../src/data-plane/server/subscription/util.ts"],"names":[],"mappings":"AACA,OAAO,MAAM,MAAM,SAAS,CAAA;AAC5B,OAAO,EAAE,YAAY,IAAI,WAAW,EAAE,MAAM,wDAAwD,CAAA;AACpG,OAAO,KAAK,OAAO,MAAM,wDAAwD,CAAA;AAIjF,qBAAa,gBAAgB;IAC3B,IAAI,EAAE,MAAM,CAAA;IACZ,UAAU,0GAA4B;gBAE1B,IAAI,EAAE;QAAE,WAAW,EAAE,MAAM,CAAA;KAAE;IAInC,GAAG,CAAC,WAAW,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC;IAOlD,OAAO;IAOb,OAAO,CAAC,YAAY;CASrB;AAED,qBAAa,WAAW;IACtB,KAAK,6FAAiC;IAEhC,GAAG,CAAC,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC;IAM7B,OAAO;CAKd;AAED;;;;;;;;;;;;;GAaG;AACH,qBAAa,eAAe,CAAC,CAAC;IAC5B,IAAI,EAAE,eAAe,CAAC,CAAC,CAAC,EAAE,CAAK;IAE/B,IAAI,CAAC,KAAK,EAAE,CAAC;IAMb,QAAQ,IAAI,CAAC,EAAE;CAOhB;AAED,qBAAa,eAAe,CAAC,CAAC;IAG1B,OAAO,CAAC,WAAW;IACZ,KAAK,EAAE,CAAC;IAHjB,UAAU,UAAQ;gBAER,WAAW,EAAE,eAAe,CAAC,CAAC,CAAC,EAChC,KAAK,EAAE,CAAC;IAGjB,QAAQ;CAIT;AAED,qBAAa,UAAU,CAAC,CAAC,EAAE,CAAC,CAAE,SAAQ,GAAG,CAAC,CAAC,EAAE,CAAC,CAAC;IAC7C,GAAG,CAAC,GAAG,EAAE,CAAC,GAAG,CAAC;CAKf;AAGD,MAAM,MAAM,kBAAkB,GAC1B,OAAO,CAAC,MAAM,GACd,OAAO,CAAC,MAAM,GACd,OAAO,CAAC,QAAQ,GAChB,OAAO,CAAC,OAAO,GACf,OAAO,CAAC,SAAS,CAAA;AAErB,wBAAgB,eAAe,CAAC,GAAG,EAAE,WAAW;;;;;;;;;;;;;EA4B/C;AAED,wBAAgB,MAAM,CAAC,KAAK,KAAA,UAE3B;AAED,wBAAgB,QAAQ,CAAC,GAAG,EAAE,MAAM,UAInC"}
|