@atproto/ozone 0.0.12 → 0.0.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +10 -0
- package/dist/api/label/queryLabels.d.ts +3 -0
- package/dist/api/label/subscribeLabels.d.ts +3 -0
- package/dist/config/config.d.ts +3 -0
- package/dist/config/env.d.ts +3 -0
- package/dist/context.d.ts +3 -0
- package/dist/db/index.js +3 -1
- package/dist/db/index.js.map +2 -2
- package/dist/db/schema/label.d.ts +4 -0
- package/dist/index.js +593 -244
- package/dist/index.js.map +3 -3
- package/dist/logger.d.ts +1 -0
- package/dist/mod-service/util.d.ts +3 -0
- package/dist/sequencer/index.d.ts +2 -0
- package/dist/sequencer/outbox.d.ts +16 -0
- package/dist/sequencer/sequencer.d.ts +33 -0
- package/package.json +10 -10
- package/src/api/admin/emitModerationEvent.ts +16 -10
- package/src/api/index.ts +4 -0
- package/src/api/label/queryLabels.ts +58 -0
- package/src/api/label/subscribeLabels.ts +25 -0
- package/src/api/temp/fetchLabels.ts +2 -4
- package/src/config/config.ts +6 -0
- package/src/config/env.ts +6 -0
- package/src/context.ts +12 -0
- package/src/db/migrations/20231219T205730722Z-init.ts +7 -1
- package/src/db/schema/label.ts +7 -0
- package/src/index.ts +2 -0
- package/src/lexicon/lexicons.ts +1 -1
- package/src/logger.ts +2 -0
- package/src/mod-service/index.ts +73 -72
- package/src/mod-service/status.ts +3 -0
- package/src/mod-service/util.ts +17 -0
- package/src/mod-service/views.ts +2 -5
- package/src/sequencer/index.ts +2 -0
- package/src/sequencer/outbox.ts +122 -0
- package/src/sequencer/sequencer.ts +143 -0
- package/tests/__snapshots__/moderation-events.test.ts.snap +53 -75
- package/tests/__snapshots__/moderation.test.ts.snap +4 -4
- package/tests/moderation-appeals.test.ts +19 -7
- package/tests/moderation-events.test.ts +7 -7
- package/tests/moderation-statuses.test.ts +2 -2
- package/tests/moderation.test.ts +14 -13
- package/tests/query-labels.test.ts +163 -0
- package/tests/repo-search.test.ts +0 -1
- package/tests/sequencer.test.ts +222 -0
- package/tests/server.test.ts +2 -0
package/dist/logger.d.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { subsystemLogger } from '@atproto/common';
|
|
2
2
|
export declare const dbLogger: ReturnType<typeof subsystemLogger>;
|
|
3
|
+
export declare const seqLogger: ReturnType<typeof subsystemLogger>;
|
|
3
4
|
export declare const httpLogger: ReturnType<typeof subsystemLogger>;
|
|
4
5
|
export declare const langLogger: ReturnType<typeof subsystemLogger>;
|
|
5
6
|
export declare const loggerMiddleware: import("pino-http").HttpLogger;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { AsyncBuffer } from '@atproto/common';
|
|
2
|
+
import { Sequencer, LabelsEvt } from './sequencer';
|
|
3
|
+
export type OutboxOpts = {
|
|
4
|
+
maxBufferSize: number;
|
|
5
|
+
};
|
|
6
|
+
export declare class Outbox {
|
|
7
|
+
sequencer: Sequencer;
|
|
8
|
+
private caughtUp;
|
|
9
|
+
lastSeen: number;
|
|
10
|
+
cutoverBuffer: LabelsEvt[];
|
|
11
|
+
outBuffer: AsyncBuffer<LabelsEvt>;
|
|
12
|
+
constructor(sequencer: Sequencer, opts?: Partial<OutboxOpts>);
|
|
13
|
+
events(backfillCursor?: number, signal?: AbortSignal): AsyncGenerator<LabelsEvt>;
|
|
14
|
+
getBackfill(backfillCursor: number): AsyncGenerator<LabelsEvt, void, unknown>;
|
|
15
|
+
}
|
|
16
|
+
export default Outbox;
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import TypedEmitter from 'typed-emitter';
|
|
2
|
+
import Database from '../db';
|
|
3
|
+
import { Labels as LabelsEvt } from '../lexicon/types/com/atproto/label/subscribeLabels';
|
|
4
|
+
import { Label as LabelTable } from '../db/schema/label';
|
|
5
|
+
import { Selectable } from 'kysely';
|
|
6
|
+
import { PoolClient } from 'pg';
|
|
7
|
+
export type { Labels as LabelsEvt } from '../lexicon/types/com/atproto/label/subscribeLabels';
|
|
8
|
+
type LabelRow = Selectable<LabelTable>;
|
|
9
|
+
declare const Sequencer_base: new () => SequencerEmitter;
|
|
10
|
+
export declare class Sequencer extends Sequencer_base {
|
|
11
|
+
db: Database;
|
|
12
|
+
lastSeen: number;
|
|
13
|
+
destroyed: boolean;
|
|
14
|
+
pollPromise: Promise<void> | undefined;
|
|
15
|
+
queued: boolean;
|
|
16
|
+
conn: PoolClient | undefined;
|
|
17
|
+
constructor(db: Database, lastSeen?: number);
|
|
18
|
+
start(): Promise<void>;
|
|
19
|
+
destroy(): Promise<void>;
|
|
20
|
+
curr(): Promise<number | null>;
|
|
21
|
+
next(cursor: number): Promise<LabelRow | null>;
|
|
22
|
+
requestLabelRange(opts: {
|
|
23
|
+
earliestId?: number;
|
|
24
|
+
limit?: number;
|
|
25
|
+
}): Promise<LabelsEvt[]>;
|
|
26
|
+
private poll;
|
|
27
|
+
}
|
|
28
|
+
type SequencerEvents = {
|
|
29
|
+
events: (evts: LabelsEvt[]) => void;
|
|
30
|
+
close: () => void;
|
|
31
|
+
};
|
|
32
|
+
export type SequencerEmitter = TypedEmitter<SequencerEvents>;
|
|
33
|
+
export default Sequencer;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@atproto/ozone",
|
|
3
|
-
"version": "0.0.
|
|
3
|
+
"version": "0.0.13",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "Backend service for moderating the Bluesky network.",
|
|
6
6
|
"keywords": [
|
|
@@ -30,13 +30,13 @@
|
|
|
30
30
|
"pino-http": "^8.2.1",
|
|
31
31
|
"typed-emitter": "^2.1.0",
|
|
32
32
|
"uint8arrays": "3.0.0",
|
|
33
|
-
"@atproto/api": "^0.10.
|
|
33
|
+
"@atproto/api": "^0.10.2",
|
|
34
34
|
"@atproto/common": "^0.3.3",
|
|
35
35
|
"@atproto/crypto": "^0.3.0",
|
|
36
36
|
"@atproto/identity": "^0.3.2",
|
|
37
|
-
"@atproto/lexicon": "^0.3.
|
|
38
|
-
"@atproto/syntax": "^0.
|
|
39
|
-
"@atproto/xrpc-server": "^0.4.
|
|
37
|
+
"@atproto/lexicon": "^0.3.2",
|
|
38
|
+
"@atproto/syntax": "^0.2.0",
|
|
39
|
+
"@atproto/xrpc-server": "^0.4.3"
|
|
40
40
|
},
|
|
41
41
|
"devDependencies": {
|
|
42
42
|
"@did-plc/server": "^0.0.1",
|
|
@@ -46,11 +46,11 @@
|
|
|
46
46
|
"@types/pg": "^8.6.6",
|
|
47
47
|
"@types/qs": "^6.9.7",
|
|
48
48
|
"axios": "^0.27.2",
|
|
49
|
-
"@atproto/api": "^0.10.
|
|
50
|
-
"@atproto/dev-env": "^0.2.
|
|
51
|
-
"@atproto/lex-cli": "^0.3.
|
|
52
|
-
"@atproto/pds": "^0.4.
|
|
53
|
-
"@atproto/xrpc": "^0.4.
|
|
49
|
+
"@atproto/api": "^0.10.2",
|
|
50
|
+
"@atproto/dev-env": "^0.2.34",
|
|
51
|
+
"@atproto/lex-cli": "^0.3.1",
|
|
52
|
+
"@atproto/pds": "^0.4.2",
|
|
53
|
+
"@atproto/xrpc": "^0.4.2"
|
|
54
54
|
},
|
|
55
55
|
"scripts": {
|
|
56
56
|
"codegen": "lex gen-server ./src/lexicon ../../lexicons/com/atproto/*/* ../../lexicons/app/bsky/*/*",
|
|
@@ -27,17 +27,23 @@ export default function (server: Server, ctx: AppContext) {
|
|
|
27
27
|
|
|
28
28
|
// apply access rules
|
|
29
29
|
|
|
30
|
-
// if less than moderator access then can not takedown an account
|
|
31
|
-
if (!access.moderator && isTakedownEvent && subject.isRepo()) {
|
|
32
|
-
throw new AuthRequiredError(
|
|
33
|
-
'Must be a full moderator to perform an account takedown',
|
|
34
|
-
)
|
|
35
|
-
}
|
|
36
30
|
// if less than moderator access then can only take ack and escalation actions
|
|
37
|
-
if (
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
|
|
31
|
+
if (isTakedownEvent || isReverseTakedownEvent) {
|
|
32
|
+
if (!access.moderator) {
|
|
33
|
+
throw new AuthRequiredError(
|
|
34
|
+
'Must be a full moderator to take this type of action',
|
|
35
|
+
)
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
// Non admins should not be able to take down feed generators
|
|
39
|
+
if (
|
|
40
|
+
!access.admin &&
|
|
41
|
+
subject.recordPath?.includes('app.bsky.feed.generator/')
|
|
42
|
+
) {
|
|
43
|
+
throw new AuthRequiredError(
|
|
44
|
+
'Must be a full admin to take this type of action on feed generators',
|
|
45
|
+
)
|
|
46
|
+
}
|
|
41
47
|
}
|
|
42
48
|
// if less than moderator access then can not apply labels
|
|
43
49
|
if (!access.moderator && isLabelEvent) {
|
package/src/api/index.ts
CHANGED
|
@@ -8,6 +8,8 @@ import getRepo from './admin/getRepo'
|
|
|
8
8
|
import queryModerationStatuses from './admin/queryModerationStatuses'
|
|
9
9
|
import queryModerationEvents from './admin/queryModerationEvents'
|
|
10
10
|
import getModerationEvent from './admin/getModerationEvent'
|
|
11
|
+
import queryLabels from './label/queryLabels'
|
|
12
|
+
import subscribeLabels from './label/subscribeLabels'
|
|
11
13
|
import fetchLabels from './temp/fetchLabels'
|
|
12
14
|
import createCommunicationTemplate from './admin/createCommunicationTemplate'
|
|
13
15
|
import updateCommunicationTemplate from './admin/updateCommunicationTemplate'
|
|
@@ -27,6 +29,8 @@ export default function (server: Server, ctx: AppContext) {
|
|
|
27
29
|
getModerationEvent(server, ctx)
|
|
28
30
|
queryModerationEvents(server, ctx)
|
|
29
31
|
queryModerationStatuses(server, ctx)
|
|
32
|
+
queryLabels(server, ctx)
|
|
33
|
+
subscribeLabels(server, ctx)
|
|
30
34
|
fetchLabels(server, ctx)
|
|
31
35
|
listCommunicationTemplates(server, ctx)
|
|
32
36
|
createCommunicationTemplate(server, ctx)
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import { Server } from '../../lexicon'
|
|
2
|
+
import AppContext from '../../context'
|
|
3
|
+
import { InvalidRequestError } from '@atproto/xrpc-server'
|
|
4
|
+
import { sql } from 'kysely'
|
|
5
|
+
import { formatLabel } from '../../mod-service/util'
|
|
6
|
+
|
|
7
|
+
export default function (server: Server, ctx: AppContext) {
|
|
8
|
+
server.com.atproto.label.queryLabels(async ({ params }) => {
|
|
9
|
+
const { uriPatterns, sources, limit, cursor } = params
|
|
10
|
+
let builder = ctx.db.db.selectFrom('label').selectAll().limit(limit)
|
|
11
|
+
// if includes '*', then we don't need a where clause
|
|
12
|
+
if (!uriPatterns.includes('*')) {
|
|
13
|
+
builder = builder.where((qb) => {
|
|
14
|
+
// starter where clause that is always false so that we can chain `orWhere`s
|
|
15
|
+
qb = qb.where(sql`1 = 0`)
|
|
16
|
+
for (const pattern of uriPatterns) {
|
|
17
|
+
// if no '*', then we're looking for an exact match
|
|
18
|
+
if (!pattern.includes('*')) {
|
|
19
|
+
qb = qb.orWhere('uri', '=', pattern)
|
|
20
|
+
} else {
|
|
21
|
+
if (pattern.indexOf('*') < pattern.length - 1) {
|
|
22
|
+
throw new InvalidRequestError(`invalid pattern: ${pattern}`)
|
|
23
|
+
}
|
|
24
|
+
const searchPattern = pattern
|
|
25
|
+
.slice(0, -1)
|
|
26
|
+
.replaceAll('%', '') // sanitize search pattern
|
|
27
|
+
.replaceAll('_', '\\_') // escape any underscores
|
|
28
|
+
qb = qb.orWhere('uri', 'like', `${searchPattern}%`)
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
return qb
|
|
32
|
+
})
|
|
33
|
+
}
|
|
34
|
+
if (sources && sources.length > 0) {
|
|
35
|
+
builder = builder.where('src', 'in', sources)
|
|
36
|
+
}
|
|
37
|
+
if (cursor) {
|
|
38
|
+
const cursorId = parseInt(cursor, 10)
|
|
39
|
+
if (isNaN(cursorId)) {
|
|
40
|
+
throw new InvalidRequestError('invalid cursor')
|
|
41
|
+
}
|
|
42
|
+
builder = builder.where('id', '>', cursorId)
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const res = await builder.execute()
|
|
46
|
+
|
|
47
|
+
const labels = res.map((l) => formatLabel(l))
|
|
48
|
+
const resCursor = res.at(-1)?.id.toString(10)
|
|
49
|
+
|
|
50
|
+
return {
|
|
51
|
+
encoding: 'application/json',
|
|
52
|
+
body: {
|
|
53
|
+
cursor: resCursor,
|
|
54
|
+
labels,
|
|
55
|
+
},
|
|
56
|
+
}
|
|
57
|
+
})
|
|
58
|
+
}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { Server } from '../../lexicon'
|
|
2
|
+
import AppContext from '../../context'
|
|
3
|
+
import Outbox from '../../sequencer/outbox'
|
|
4
|
+
import { InvalidRequestError } from '@atproto/xrpc-server'
|
|
5
|
+
|
|
6
|
+
export default function (server: Server, ctx: AppContext) {
|
|
7
|
+
server.com.atproto.label.subscribeLabels(async function* ({
|
|
8
|
+
params,
|
|
9
|
+
signal,
|
|
10
|
+
}) {
|
|
11
|
+
const { cursor } = params
|
|
12
|
+
const outbox = new Outbox(ctx.sequencer)
|
|
13
|
+
|
|
14
|
+
if (cursor !== undefined) {
|
|
15
|
+
const curr = await ctx.sequencer.curr()
|
|
16
|
+
if (cursor > (curr ?? 0)) {
|
|
17
|
+
throw new InvalidRequestError('Cursor in the future.', 'FutureCursor')
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
for await (const evt of outbox.events(cursor, signal)) {
|
|
22
|
+
yield { $type: 'com.atproto.label.subscribeLabels#labels', ...evt }
|
|
23
|
+
}
|
|
24
|
+
})
|
|
25
|
+
}
|
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { Server } from '../../lexicon'
|
|
2
2
|
import AppContext from '../../context'
|
|
3
|
+
import { formatLabel } from '../../mod-service/util'
|
|
3
4
|
import {
|
|
4
5
|
UNSPECCED_TAKEDOWN_BLOBS_LABEL,
|
|
5
6
|
UNSPECCED_TAKEDOWN_LABEL,
|
|
@@ -28,10 +29,7 @@ export default function (server: Server, ctx: AppContext) {
|
|
|
28
29
|
.limit(limit)
|
|
29
30
|
.execute()
|
|
30
31
|
|
|
31
|
-
const labels = labelRes.map((l) => (
|
|
32
|
-
...l,
|
|
33
|
-
cid: l.cid === '' ? undefined : l.cid,
|
|
34
|
-
}))
|
|
32
|
+
const labels = labelRes.map((l) => formatLabel(l))
|
|
35
33
|
|
|
36
34
|
return {
|
|
37
35
|
encoding: 'application/json',
|
package/src/config/config.ts
CHANGED
|
@@ -19,6 +19,9 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => {
|
|
|
19
19
|
const dbCfg: OzoneConfig['db'] = {
|
|
20
20
|
postgresUrl: env.dbPostgresUrl,
|
|
21
21
|
postgresSchema: env.dbPostgresSchema,
|
|
22
|
+
poolSize: env.dbPoolSize,
|
|
23
|
+
poolMaxUses: env.dbPoolMaxUses,
|
|
24
|
+
poolIdleTimeoutMs: env.dbPoolIdleTimeoutMs,
|
|
22
25
|
}
|
|
23
26
|
|
|
24
27
|
assert(env.appviewUrl)
|
|
@@ -67,6 +70,9 @@ export type ServiceConfig = {
|
|
|
67
70
|
export type DatabaseConfig = {
|
|
68
71
|
postgresUrl: string
|
|
69
72
|
postgresSchema?: string
|
|
73
|
+
poolSize?: number
|
|
74
|
+
poolMaxUses?: number
|
|
75
|
+
poolIdleTimeoutMs?: number
|
|
70
76
|
}
|
|
71
77
|
|
|
72
78
|
export type AppviewConfig = {
|
package/src/config/env.ts
CHANGED
|
@@ -13,6 +13,9 @@ export const readEnv = (): OzoneEnvironment => {
|
|
|
13
13
|
pdsDid: envStr('OZONE_PDS_DID'),
|
|
14
14
|
dbPostgresUrl: envStr('OZONE_DB_POSTGRES_URL'),
|
|
15
15
|
dbPostgresSchema: envStr('OZONE_DB_POSTGRES_SCHEMA'),
|
|
16
|
+
dbPoolSize: envInt('OZONE_DB_POOL_SIZE'),
|
|
17
|
+
dbPoolMaxUses: envInt('OZONE_DB_POOL_MAX_USES'),
|
|
18
|
+
dbPoolIdleTimeoutMs: envInt('OZONE_DB_POOL_IDLE_TIMEOUT_MS'),
|
|
16
19
|
didPlcUrl: envStr('OZONE_DID_PLC_URL'),
|
|
17
20
|
adminPassword: envStr('OZONE_ADMIN_PASSWORD'),
|
|
18
21
|
moderatorPassword: envStr('OZONE_MODERATOR_PASSWORD'),
|
|
@@ -33,6 +36,9 @@ export type OzoneEnvironment = {
|
|
|
33
36
|
pdsDid?: string
|
|
34
37
|
dbPostgresUrl?: string
|
|
35
38
|
dbPostgresSchema?: string
|
|
39
|
+
dbPoolSize?: number
|
|
40
|
+
dbPoolMaxUses?: number
|
|
41
|
+
dbPoolIdleTimeoutMs?: number
|
|
36
42
|
didPlcUrl?: string
|
|
37
43
|
adminPassword?: string
|
|
38
44
|
moderatorPassword?: string
|
package/src/context.ts
CHANGED
|
@@ -10,6 +10,7 @@ import * as auth from './auth'
|
|
|
10
10
|
import { BackgroundQueue } from './background'
|
|
11
11
|
import assert from 'assert'
|
|
12
12
|
import { EventPusher } from './daemon'
|
|
13
|
+
import Sequencer from './sequencer/sequencer'
|
|
13
14
|
import {
|
|
14
15
|
CommunicationTemplateService,
|
|
15
16
|
CommunicationTemplateServiceCreator,
|
|
@@ -25,6 +26,7 @@ export type AppContextOptions = {
|
|
|
25
26
|
signingKey: Keypair
|
|
26
27
|
idResolver: IdResolver
|
|
27
28
|
backgroundQueue: BackgroundQueue
|
|
29
|
+
sequencer: Sequencer
|
|
28
30
|
}
|
|
29
31
|
|
|
30
32
|
export class AppContext {
|
|
@@ -38,6 +40,9 @@ export class AppContext {
|
|
|
38
40
|
const db = new Database({
|
|
39
41
|
url: cfg.db.postgresUrl,
|
|
40
42
|
schema: cfg.db.postgresSchema,
|
|
43
|
+
poolSize: cfg.db.poolSize,
|
|
44
|
+
poolMaxUses: cfg.db.poolMaxUses,
|
|
45
|
+
poolIdleTimeoutMs: cfg.db.poolIdleTimeoutMs,
|
|
41
46
|
})
|
|
42
47
|
const signingKey = await Secp256k1Keypair.import(secrets.signingKeyHex)
|
|
43
48
|
const appviewAgent = new AtpAgent({ service: cfg.appview.url })
|
|
@@ -74,6 +79,8 @@ export class AppContext {
|
|
|
74
79
|
plcUrl: cfg.identity.plcUrl,
|
|
75
80
|
})
|
|
76
81
|
|
|
82
|
+
const sequencer = new Sequencer(db)
|
|
83
|
+
|
|
77
84
|
return new AppContext(
|
|
78
85
|
{
|
|
79
86
|
db,
|
|
@@ -85,6 +92,7 @@ export class AppContext {
|
|
|
85
92
|
signingKey,
|
|
86
93
|
idResolver,
|
|
87
94
|
backgroundQueue,
|
|
95
|
+
sequencer,
|
|
88
96
|
...(overrides ?? {}),
|
|
89
97
|
},
|
|
90
98
|
secrets,
|
|
@@ -139,6 +147,10 @@ export class AppContext {
|
|
|
139
147
|
return this.opts.backgroundQueue
|
|
140
148
|
}
|
|
141
149
|
|
|
150
|
+
get sequencer(): Sequencer {
|
|
151
|
+
return this.opts.sequencer
|
|
152
|
+
}
|
|
153
|
+
|
|
142
154
|
get authVerifier() {
|
|
143
155
|
return auth.authVerifier(this.idResolver, { aud: this.cfg.service.did })
|
|
144
156
|
}
|
|
@@ -68,13 +68,19 @@ export async function up(db: Kysely<unknown>): Promise<void> {
|
|
|
68
68
|
// Label
|
|
69
69
|
await db.schema
|
|
70
70
|
.createTable('label')
|
|
71
|
+
.addColumn('id', 'bigserial', (col) => col.primaryKey())
|
|
71
72
|
.addColumn('src', 'varchar', (col) => col.notNull())
|
|
72
73
|
.addColumn('uri', 'varchar', (col) => col.notNull())
|
|
73
74
|
.addColumn('cid', 'varchar', (col) => col.notNull())
|
|
74
75
|
.addColumn('val', 'varchar', (col) => col.notNull())
|
|
75
76
|
.addColumn('neg', 'boolean', (col) => col.notNull())
|
|
76
77
|
.addColumn('cts', 'varchar', (col) => col.notNull())
|
|
77
|
-
.
|
|
78
|
+
.execute()
|
|
79
|
+
await db.schema
|
|
80
|
+
.createIndex('unique_label_idx')
|
|
81
|
+
.unique()
|
|
82
|
+
.on('label')
|
|
83
|
+
.columns(['src', 'uri', 'cid', 'val'])
|
|
78
84
|
.execute()
|
|
79
85
|
await db.schema
|
|
80
86
|
.createIndex('label_uri_index')
|
package/src/db/schema/label.ts
CHANGED
|
@@ -1,6 +1,9 @@
|
|
|
1
|
+
import { Generated, Selectable } from 'kysely'
|
|
2
|
+
|
|
1
3
|
export const tableName = 'label'
|
|
2
4
|
|
|
3
5
|
export interface Label {
|
|
6
|
+
id: Generated<number>
|
|
4
7
|
src: string
|
|
5
8
|
uri: string
|
|
6
9
|
cid: string
|
|
@@ -9,4 +12,8 @@ export interface Label {
|
|
|
9
12
|
cts: string
|
|
10
13
|
}
|
|
11
14
|
|
|
15
|
+
export type LabelRow = Selectable<Label>
|
|
16
|
+
|
|
12
17
|
export type PartialDB = { [tableName]: Label }
|
|
18
|
+
|
|
19
|
+
export const LabelChannel = 'label_channel' // used with notify/listen
|
package/src/index.ts
CHANGED
|
@@ -81,6 +81,7 @@ export class OzoneService {
|
|
|
81
81
|
'background queue stats',
|
|
82
82
|
)
|
|
83
83
|
}, 10000)
|
|
84
|
+
await this.ctx.sequencer.start()
|
|
84
85
|
const server = this.app.listen(this.ctx.cfg.service.port)
|
|
85
86
|
this.server = server
|
|
86
87
|
server.keepAliveTimeout = 90000
|
|
@@ -94,6 +95,7 @@ export class OzoneService {
|
|
|
94
95
|
async destroy(): Promise<void> {
|
|
95
96
|
await this.terminator?.terminate()
|
|
96
97
|
await this.ctx.backgroundQueue.destroy()
|
|
98
|
+
await this.ctx.sequencer.destroy()
|
|
97
99
|
await this.ctx.db.close()
|
|
98
100
|
clearInterval(this.dbStatsInterval)
|
|
99
101
|
}
|
package/src/lexicon/lexicons.ts
CHANGED
|
@@ -4867,7 +4867,7 @@ export const schemaDict = {
|
|
|
4867
4867
|
main: {
|
|
4868
4868
|
type: 'query',
|
|
4869
4869
|
description:
|
|
4870
|
-
'Fetch all labels from a labeler created after a certain date.
|
|
4870
|
+
'DEPRECATED: use queryLabels or subscribeLabels instead -- Fetch all labels from a labeler created after a certain date.',
|
|
4871
4871
|
parameters: {
|
|
4872
4872
|
type: 'params',
|
|
4873
4873
|
properties: {
|
package/src/logger.ts
CHANGED
|
@@ -3,6 +3,8 @@ import { subsystemLogger } from '@atproto/common'
|
|
|
3
3
|
|
|
4
4
|
export const dbLogger: ReturnType<typeof subsystemLogger> =
|
|
5
5
|
subsystemLogger('ozone:db')
|
|
6
|
+
export const seqLogger: ReturnType<typeof subsystemLogger> =
|
|
7
|
+
subsystemLogger('ozone:sequencer')
|
|
6
8
|
export const httpLogger: ReturnType<typeof subsystemLogger> =
|
|
7
9
|
subsystemLogger('ozone')
|
|
8
10
|
export const langLogger: ReturnType<typeof subsystemLogger> =
|
package/src/mod-service/index.ts
CHANGED
|
@@ -43,6 +43,7 @@ import { BlobPushEvent } from '../db/schema/blob_push_event'
|
|
|
43
43
|
import { BackgroundQueue } from '../background'
|
|
44
44
|
import { EventPusher } from '../daemon'
|
|
45
45
|
import { jsonb } from '../db/types'
|
|
46
|
+
import { LabelChannel } from '../db/schema/label'
|
|
46
47
|
|
|
47
48
|
export type ModerationServiceCreator = (db: Database) => ModerationService
|
|
48
49
|
|
|
@@ -438,24 +439,22 @@ export class ModerationService {
|
|
|
438
439
|
takedownRef,
|
|
439
440
|
}))
|
|
440
441
|
|
|
441
|
-
const
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
.
|
|
446
|
-
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
}),
|
|
458
|
-
])
|
|
442
|
+
const repoEvts = await this.db.db
|
|
443
|
+
.insertInto('repo_push_event')
|
|
444
|
+
.values(values)
|
|
445
|
+
.onConflict((oc) =>
|
|
446
|
+
oc.columns(['subjectDid', 'eventType']).doUpdateSet({
|
|
447
|
+
takedownRef,
|
|
448
|
+
confirmedAt: null,
|
|
449
|
+
attempts: 0,
|
|
450
|
+
lastAttempted: null,
|
|
451
|
+
}),
|
|
452
|
+
)
|
|
453
|
+
.returning('id')
|
|
454
|
+
.execute()
|
|
455
|
+
await this.formatAndCreateLabels(subject.did, null, {
|
|
456
|
+
create: [UNSPECCED_TAKEDOWN_LABEL],
|
|
457
|
+
})
|
|
459
458
|
|
|
460
459
|
this.db.onCommit(() => {
|
|
461
460
|
this.backgroundQueue.add(async () => {
|
|
@@ -467,23 +466,21 @@ export class ModerationService {
|
|
|
467
466
|
}
|
|
468
467
|
|
|
469
468
|
async reverseTakedownRepo(subject: RepoSubject) {
|
|
470
|
-
const
|
|
471
|
-
|
|
472
|
-
|
|
473
|
-
|
|
474
|
-
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
}),
|
|
486
|
-
])
|
|
469
|
+
const repoEvts = await this.db.db
|
|
470
|
+
.updateTable('repo_push_event')
|
|
471
|
+
.where('eventType', 'in', TAKEDOWNS)
|
|
472
|
+
.where('subjectDid', '=', subject.did)
|
|
473
|
+
.set({
|
|
474
|
+
takedownRef: null,
|
|
475
|
+
confirmedAt: null,
|
|
476
|
+
attempts: 0,
|
|
477
|
+
lastAttempted: null,
|
|
478
|
+
})
|
|
479
|
+
.returning('id')
|
|
480
|
+
.execute()
|
|
481
|
+
await this.formatAndCreateLabels(subject.did, null, {
|
|
482
|
+
negate: [UNSPECCED_TAKEDOWN_LABEL],
|
|
483
|
+
})
|
|
487
484
|
|
|
488
485
|
this.db.onCommit(() => {
|
|
489
486
|
this.backgroundQueue.add(async () => {
|
|
@@ -509,22 +506,22 @@ export class ModerationService {
|
|
|
509
506
|
if (blobCids && blobCids.length > 0) {
|
|
510
507
|
labels.push(UNSPECCED_TAKEDOWN_BLOBS_LABEL)
|
|
511
508
|
}
|
|
512
|
-
const
|
|
513
|
-
|
|
514
|
-
|
|
515
|
-
|
|
516
|
-
.
|
|
517
|
-
|
|
518
|
-
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
522
|
-
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
|
|
526
|
-
|
|
527
|
-
|
|
509
|
+
const recordEvts = await this.db.db
|
|
510
|
+
.insertInto('record_push_event')
|
|
511
|
+
.values(values)
|
|
512
|
+
.onConflict((oc) =>
|
|
513
|
+
oc.columns(['subjectUri', 'eventType']).doUpdateSet({
|
|
514
|
+
takedownRef,
|
|
515
|
+
confirmedAt: null,
|
|
516
|
+
attempts: 0,
|
|
517
|
+
lastAttempted: null,
|
|
518
|
+
}),
|
|
519
|
+
)
|
|
520
|
+
.returning('id')
|
|
521
|
+
.execute()
|
|
522
|
+
await this.formatAndCreateLabels(subject.uri, subject.cid, {
|
|
523
|
+
create: labels,
|
|
524
|
+
})
|
|
528
525
|
|
|
529
526
|
this.db.onCommit(() => {
|
|
530
527
|
this.backgroundQueue.add(async () => {
|
|
@@ -579,29 +576,31 @@ export class ModerationService {
|
|
|
579
576
|
if (blobCids && blobCids.length > 0) {
|
|
580
577
|
labels.push(UNSPECCED_TAKEDOWN_BLOBS_LABEL)
|
|
581
578
|
}
|
|
582
|
-
const
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
579
|
+
const recordEvts = await this.db.db
|
|
580
|
+
.updateTable('record_push_event')
|
|
581
|
+
.where('eventType', 'in', TAKEDOWNS)
|
|
582
|
+
.where('subjectDid', '=', subject.did)
|
|
583
|
+
.where('subjectUri', '=', subject.uri)
|
|
584
|
+
.set({
|
|
585
|
+
takedownRef: null,
|
|
586
|
+
confirmedAt: null,
|
|
587
|
+
attempts: 0,
|
|
588
|
+
lastAttempted: null,
|
|
589
|
+
})
|
|
590
|
+
.returning('id')
|
|
591
|
+
.execute()
|
|
592
|
+
await this.formatAndCreateLabels(subject.uri, subject.cid, {
|
|
593
|
+
negate: labels,
|
|
594
|
+
}),
|
|
595
|
+
this.db.onCommit(() => {
|
|
596
|
+
this.backgroundQueue.add(async () => {
|
|
597
|
+
await Promise.all(
|
|
598
|
+
recordEvts.map((evt) =>
|
|
599
|
+
this.eventPusher.attemptRecordEvent(evt.id),
|
|
600
|
+
),
|
|
601
|
+
)
|
|
593
602
|
})
|
|
594
|
-
.returning('id')
|
|
595
|
-
.execute(),
|
|
596
|
-
this.formatAndCreateLabels(subject.uri, subject.cid, { negate: labels }),
|
|
597
|
-
])
|
|
598
|
-
this.db.onCommit(() => {
|
|
599
|
-
this.backgroundQueue.add(async () => {
|
|
600
|
-
await Promise.all(
|
|
601
|
-
recordEvts.map((evt) => this.eventPusher.attemptRecordEvent(evt.id)),
|
|
602
|
-
)
|
|
603
603
|
})
|
|
604
|
-
})
|
|
605
604
|
|
|
606
605
|
if (blobCids && blobCids.length > 0) {
|
|
607
606
|
const blobEvts = await this.db.db
|
|
@@ -858,12 +857,14 @@ export class ModerationService {
|
|
|
858
857
|
neg: !!l.neg,
|
|
859
858
|
}))
|
|
860
859
|
const { ref } = this.db.db.dynamic
|
|
860
|
+
await sql`notify ${ref(LabelChannel)}`.execute(this.db.db)
|
|
861
861
|
const excluded = (col: string) => ref(`excluded.${col}`)
|
|
862
862
|
await this.db.db
|
|
863
863
|
.insertInto('label')
|
|
864
864
|
.values(dbVals)
|
|
865
865
|
.onConflict((oc) =>
|
|
866
866
|
oc.columns(['src', 'uri', 'cid', 'val']).doUpdateSet({
|
|
867
|
+
id: sql`${excluded('id')}`,
|
|
867
868
|
neg: sql`${excluded('neg')}`,
|
|
868
869
|
cts: sql`${excluded('cts')}`,
|
|
869
870
|
}),
|
|
@@ -179,6 +179,9 @@ export const adjustModerationSubjectStatus = async (
|
|
|
179
179
|
subjectStatus.appealed = true
|
|
180
180
|
newStatus.lastAppealedAt = createdAt
|
|
181
181
|
subjectStatus.lastAppealedAt = createdAt
|
|
182
|
+
// Set reviewState to escalated when appeal events are emitted
|
|
183
|
+
subjectStatus.reviewState = REVIEWESCALATED
|
|
184
|
+
newStatus.reviewState = REVIEWESCALATED
|
|
182
185
|
}
|
|
183
186
|
|
|
184
187
|
if (
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
import { LabelRow } from '../db/schema/label'
|
|
2
|
+
import { Label } from '../lexicon/types/com/atproto/label/defs'
|
|
3
|
+
|
|
4
|
+
export const formatLabel = (row: LabelRow): Label => {
|
|
5
|
+
const label: Label = {
|
|
6
|
+
src: row.src,
|
|
7
|
+
uri: row.uri,
|
|
8
|
+
val: row.val,
|
|
9
|
+
neg: row.neg,
|
|
10
|
+
cts: row.cts,
|
|
11
|
+
}
|
|
12
|
+
if (row.cid !== '') {
|
|
13
|
+
// @NOTE avoiding undefined values on label, which dag-cbor chokes on when serializing.
|
|
14
|
+
label.cid = row.cid
|
|
15
|
+
}
|
|
16
|
+
return label
|
|
17
|
+
}
|