@atproto/pds 0.3.0-beta.2 → 0.3.0-beta.3

package/dist/lexicon/index.d.ts

@@ -57,6 +57,7 @@ import * as ComAtprotoSyncListRepos from './types/com/atproto/sync/listRepos';
 import * as ComAtprotoSyncNotifyOfUpdate from './types/com/atproto/sync/notifyOfUpdate';
 import * as ComAtprotoSyncRequestCrawl from './types/com/atproto/sync/requestCrawl';
 import * as ComAtprotoSyncSubscribeRepos from './types/com/atproto/sync/subscribeRepos';
+import * as ComAtprotoTempUpgradeRepoVersion from './types/com/atproto/temp/upgradeRepoVersion';
 import * as AppBskyActorGetPreferences from './types/app/bsky/actor/getPreferences';
 import * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile';
 import * as AppBskyActorGetProfiles from './types/app/bsky/actor/getProfiles';
@@ -137,6 +138,7 @@ export declare class AtprotoNS {
     repo: RepoNS;
     server: ServerNS;
     sync: SyncNS;
+    temp: TempNS;
     constructor(server: Server);
 }
 export declare class AdminNS {
@@ -225,6 +227,11 @@ export declare class SyncNS {
     requestCrawl<AV extends AuthVerifier>(cfg: ConfigOf<AV, ComAtprotoSyncRequestCrawl.Handler<ExtractAuth<AV>>, ComAtprotoSyncRequestCrawl.HandlerReqCtx<ExtractAuth<AV>>>): void;
     subscribeRepos<AV extends StreamAuthVerifier>(cfg: ConfigOf<AV, ComAtprotoSyncSubscribeRepos.Handler<ExtractAuth<AV>>, ComAtprotoSyncSubscribeRepos.HandlerReqCtx<ExtractAuth<AV>>>): void;
 }
+export declare class TempNS {
+    _server: Server;
+    constructor(server: Server);
+    upgradeRepoVersion<AV extends AuthVerifier>(cfg: ConfigOf<AV, ComAtprotoTempUpgradeRepoVersion.Handler<ExtractAuth<AV>>, ComAtprotoTempUpgradeRepoVersion.HandlerReqCtx<ExtractAuth<AV>>>): void;
+}
 export declare class AppNS {
     _server: Server;
     bsky: BskyNS;

package/dist/lexicon/lexicons.d.ts

@@ -3251,6 +3251,32 @@ export declare const schemaDict: {
             };
         };
     };
+    ComAtprotoTempUpgradeRepoVersion: {
+        lexicon: number;
+        id: string;
+        defs: {
+            main: {
+                type: string;
+                description: string;
+                input: {
+                    encoding: string;
+                    schema: {
+                        type: string;
+                        required: string[];
+                        properties: {
+                            did: {
+                                type: string;
+                                format: string;
+                            };
+                            force: {
+                                type: string;
+                            };
+                        };
+                    };
+                };
+            };
+        };
+    };
     AppBskyActorDefs: {
         lexicon: number;
         id: string;
@@ -6465,6 +6491,7 @@ export declare const ids: {
     ComAtprotoSyncNotifyOfUpdate: string;
     ComAtprotoSyncRequestCrawl: string;
     ComAtprotoSyncSubscribeRepos: string;
+    ComAtprotoTempUpgradeRepoVersion: string;
     AppBskyActorDefs: string;
     AppBskyActorGetPreferences: string;
     AppBskyActorGetProfile: string;

package/dist/services/index.d.ts

@@ -13,6 +13,7 @@ import { LocalService } from './local';
 export declare function createServices(resources: {
     repoSigningKey: crypto.Keypair;
     blobstore: BlobStore;
+    pdsHostname: string;
     appViewAgent?: AtpAgent;
     appViewDid?: string;
     appViewCdnUrlPattern?: string;

package/dist/services/local/index.d.ts

@@ -15,11 +15,12 @@ declare type CommonSignedUris = 'avatar' | 'banner' | 'feed_thumbnail' | 'feed_f
 export declare class LocalService {
     db: Database;
     signingKey: Keypair;
+    pdsHostname: string;
     appviewAgent?: AtpAgent | undefined;
     appviewDid?: string | undefined;
     appviewCdnUrlPattern?: string | undefined;
-    constructor(db: Database, signingKey: Keypair, appviewAgent?: AtpAgent | undefined, appviewDid?: string | undefined, appviewCdnUrlPattern?: string | undefined);
-    static creator(signingKey: Keypair, appviewAgent?: AtpAgent, appviewDid?: string, appviewCdnUrlPattern?: string): (db: Database) => LocalService;
+    constructor(db: Database, signingKey: Keypair, pdsHostname: string, appviewAgent?: AtpAgent | undefined, appviewDid?: string | undefined, appviewCdnUrlPattern?: string | undefined);
+    static creator(signingKey: Keypair, pdsHostname: string, appviewAgent?: AtpAgent, appviewDid?: string, appviewCdnUrlPattern?: string): (db: Database) => LocalService;
     getImageUrl(pattern: CommonSignedUris, did: string, cid: string): string;
     serviceAuthHeaders(did: string): Promise<{
         headers: {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/pds",
-  "version": "0.3.0-beta.2",
+  "version": "0.3.0-beta.3",
   "license": "MIT",
   "repository": {
     "type": "git",
@@ -39,15 +39,15 @@
     "typed-emitter": "^2.1.0",
     "uint8arrays": "3.0.0",
     "zod": "^3.21.4",
-    "@atproto/api": "^0.6.12",
-    "@atproto/common": "^0.3.0",
     "@atproto/aws": "^0.1.0",
-    "@atproto/crypto": "^0.2.2",
+    "@atproto/api": "^0.6.12",
     "@atproto/syntax": "^0.1.0",
+    "@atproto/common": "^0.3.0",
     "@atproto/identity": "^0.2.0",
+    "@atproto/crypto": "^0.2.2",
     "@atproto/repo": "^0.3.0",
-    "@atproto/lexicon": "^0.2.0",
     "@atproto/xrpc-server": "^0.3.0",
+    "@atproto/lexicon": "^0.2.0",
     "@atproto/xrpc": "^0.3.0"
   },
   "devDependencies": {
@@ -61,10 +61,10 @@
     "@types/qs": "^6.9.7",
     "@types/sharp": "^0.31.0",
     "axios": "^0.27.2",
-    "@atproto/bsky": "^0.0.3",
-    "@atproto/api": "^0.6.12",
     "@atproto/dev-env": "^0.2.3",
-    "@atproto/lex-cli": "^0.2.0"
+    "@atproto/api": "^0.6.12",
+    "@atproto/lex-cli": "^0.2.0",
+    "@atproto/bsky": "^0.0.3"
   },
   "scripts": {
     "codegen": "lex gen-server ./src/lexicon ../../lexicons/com/atproto/*/* ../../lexicons/app/bsky/*/*",

package/src/api/com/atproto/index.ts

@@ -6,6 +6,7 @@ import moderation from './moderation'
 import repo from './repo'
 import serverMethods from './server'
 import sync from './sync'
+import upgradeRepoVersion from './upgradeRepoVersion'
 
 export default function (server: Server, ctx: AppContext) {
   admin(server, ctx)
@@ -14,4 +15,5 @@ export default function (server: Server, ctx: AppContext) {
   repo(server, ctx)
   serverMethods(server, ctx)
   sync(server, ctx)
+  upgradeRepoVersion(server, ctx)
 }

package/src/api/com/atproto/upgradeRepoVersion.ts

@@ -0,0 +1,140 @@
+import { InvalidRequestError } from '@atproto/xrpc-server'
+import { TID, chunkArray, wait } from '@atproto/common'
+import { Server } from '../../../lexicon'
+import SqlRepoStorage from '../../../sql-repo-storage'
+import AppContext from '../../../context'
+import {
+  BlockMap,
+  CidSet,
+  DataDiff,
+  MST,
+  MemoryBlockstore,
+  def,
+  signCommit,
+} from '@atproto/repo'
+import { CID } from 'multiformats/cid'
+import { formatSeqCommit, sequenceEvt } from '../../../sequencer'
+import { httpLogger as log } from '../../../logger'
+
+export default function (server: Server, ctx: AppContext) {
+  server.com.atproto.temp.upgradeRepoVersion({
+    auth: ctx.roleVerifier,
+    handler: async ({ input, auth }) => {
+      if (!auth.credentials.admin) {
+        throw new InvalidRequestError('must be admin')
+      }
+      const { did, force } = input.body
+
+      await ctx.db.transaction(async (dbTxn) => {
+        const storage = new SqlRepoStorage(dbTxn, did)
+        await obtainLock(storage)
+        const prevCid = await storage.getRoot()
+        if (!prevCid) {
+          throw new InvalidRequestError('Could not find repo')
+        }
+        const prev = await storage.readObj(prevCid, def.versionedCommit)
+        const records = await dbTxn.db
+          .selectFrom('record')
+          .select(['collection', 'rkey', 'cid'])
+          .where('did', '=', did)
+          .execute()
+        const memoryStore = new MemoryBlockstore()
+        let data = await MST.create(memoryStore)
+        for (const record of records) {
+          const dataKey = record.collection + '/' + record.rkey
+          const cid = CID.parse(record.cid)
+          data = await data.add(dataKey, cid)
+        }
+        const dataCid = await data.getPointer()
+        if (!force && !dataCid.equals(prev.data)) {
+          throw new InvalidRequestError('Data cid did not match')
+        }
+        const recordCids = records.map((r) => r.cid)
+        const diff = await DataDiff.of(data, null)
+        const cidsToKeep = [...recordCids, ...diff.newMstBlocks.cids()]
+        const rev = TID.nextStr(prev.rev)
+        if (force) {
+          const got = await storage.getBlocks(diff.newMstBlocks.cids())
+          const toAdd = diff.newMstBlocks.getMany(got.missing)
+          log.info(
+            { missing: got.missing.length },
+            'force added missing blocks',
+          )
+          // puts any new blocks & no-ops for already existing
+          await storage.putMany(toAdd.blocks, rev)
+        }
+        for (const chunk of chunkArray(cidsToKeep, 500)) {
+          const cidStrs = chunk.map((c) => c.toString())
+          await dbTxn.db
+            .updateTable('ipld_block')
+            .set({ repoRev: rev })
+            .where('creator', '=', did)
+            .where('cid', 'in', cidStrs)
+            .execute()
+        }
+        await dbTxn.db
+          .deleteFrom('ipld_block')
+          .where('creator', '=', did)
+          .where((qb) =>
+            qb.where('repoRev', 'is', null).orWhere('repoRev', '!=', rev),
+          )
+          .execute()
+        await dbTxn.db
+          .updateTable('repo_blob')
+          .set({ repoRev: rev })
+          .where('did', '=', did)
+          .execute()
+        await dbTxn.db
+          .updateTable('record')
+          .set({ repoRev: rev })
+          .where('did', '=', did)
+          .execute()
+        const commit = await signCommit(
+          {
+            did,
+            version: 3,
+            rev: TID.nextStr(),
+            prev: prevCid,
+            data: dataCid,
+          },
+          ctx.repoSigningKey,
+        )
+        const newBlocks = new BlockMap()
+        const commitCid = await newBlocks.add(commit)
+        await storage.putMany(newBlocks, rev)
+        await dbTxn.db
+          .updateTable('repo_root')
+          .set({
+            root: commitCid.toString(),
+            rev,
+            indexedAt: storage.getTimestamp(),
+          })
+          .where('did', '=', did)
+          .execute()
+
+        const commitData = {
+          cid: commitCid,
+          rev,
+          prev: prevCid,
+          since: null,
+          newBlocks,
+          removedCids: new CidSet(),
+        }
+        const seqEvt = await formatSeqCommit(did, commitData, [])
+        await sequenceEvt(dbTxn, seqEvt)
+      })
+    },
+  })
+}
+
+const obtainLock = async (storage: SqlRepoStorage, tries = 20) => {
+  const obtained = await storage.lockRepo()
+  if (obtained) {
+    return
+  }
+  if (tries < 1) {
+    throw new InvalidRequestError('could not obtain lock')
+  }
+  await wait(50)
+  return obtainLock(storage, tries - 1)
+}

package/src/context.ts

@@ -180,6 +180,7 @@ export class AppContext {
       repoSigningKey,
       blobstore,
       appViewAgent,
+      pdsHostname: cfg.service.hostname,
       appViewDid: cfg.bskyAppView.did,
       appViewCdnUrlPattern: cfg.bskyAppView.cdnUrlPattern,
       backgroundQueue,

package/src/lexicon/index.ts

@@ -67,6 +67,7 @@ import * as ComAtprotoSyncListRepos from './types/com/atproto/sync/listRepos'
 import * as ComAtprotoSyncNotifyOfUpdate from './types/com/atproto/sync/notifyOfUpdate'
 import * as ComAtprotoSyncRequestCrawl from './types/com/atproto/sync/requestCrawl'
 import * as ComAtprotoSyncSubscribeRepos from './types/com/atproto/sync/subscribeRepos'
+import * as ComAtprotoTempUpgradeRepoVersion from './types/com/atproto/temp/upgradeRepoVersion'
 import * as AppBskyActorGetPreferences from './types/app/bsky/actor/getPreferences'
 import * as AppBskyActorGetProfile from './types/app/bsky/actor/getProfile'
 import * as AppBskyActorGetProfiles from './types/app/bsky/actor/getProfiles'
@@ -163,6 +164,7 @@ export class AtprotoNS {
   repo: RepoNS
   server: ServerNS
   sync: SyncNS
+  temp: TempNS
 
   constructor(server: Server) {
     this._server = server
@@ -173,6 +175,7 @@ export class AtprotoNS {
     this.repo = new RepoNS(server)
     this.server = new ServerNS(server)
     this.sync = new SyncNS(server)
+    this.temp = new TempNS(server)
   }
 }
 
@@ -870,6 +873,25 @@ export class SyncNS {
   }
 }
 
+export class TempNS {
+  _server: Server
+
+  constructor(server: Server) {
+    this._server = server
+  }
+
+  upgradeRepoVersion<AV extends AuthVerifier>(
+    cfg: ConfigOf<
+      AV,
+      ComAtprotoTempUpgradeRepoVersion.Handler<ExtractAuth<AV>>,
+      ComAtprotoTempUpgradeRepoVersion.HandlerReqCtx<ExtractAuth<AV>>
+    >,
+  ) {
+    const nsid = 'com.atproto.temp.upgradeRepoVersion' // @ts-ignore
+    return this._server.xrpc.method(nsid, cfg)
+  }
+}
+
 export class AppNS {
   _server: Server
   bsky: BskyNS

package/src/lexicon/lexicons.ts

@@ -3517,6 +3517,32 @@ export const schemaDict = {
       },
     },
   },
+  ComAtprotoTempUpgradeRepoVersion: {
+    lexicon: 1,
+    id: 'com.atproto.temp.upgradeRepoVersion',
+    defs: {
+      main: {
+        type: 'procedure',
+        description: 'Upgrade a repo to v3',
+        input: {
+          encoding: 'application/json',
+          schema: {
+            type: 'object',
+            required: ['did'],
+            properties: {
+              did: {
+                type: 'string',
+                format: 'did',
+              },
+              force: {
+                type: 'boolean',
+              },
+            },
+          },
+        },
+      },
+    },
+  },
   AppBskyActorDefs: {
     lexicon: 1,
     id: 'app.bsky.actor.defs',
@@ -6838,6 +6864,7 @@ export const ids = {
   ComAtprotoSyncNotifyOfUpdate: 'com.atproto.sync.notifyOfUpdate',
   ComAtprotoSyncRequestCrawl: 'com.atproto.sync.requestCrawl',
   ComAtprotoSyncSubscribeRepos: 'com.atproto.sync.subscribeRepos',
+  ComAtprotoTempUpgradeRepoVersion: 'com.atproto.temp.upgradeRepoVersion',
   AppBskyActorDefs: 'app.bsky.actor.defs',
   AppBskyActorGetPreferences: 'app.bsky.actor.getPreferences',
   AppBskyActorGetProfile: 'app.bsky.actor.getProfile',

package/src/lexicon/types/com/atproto/temp/upgradeRepoVersion.ts

@@ -0,0 +1,39 @@
+/**
+ * GENERATED CODE - DO NOT MODIFY
+ */
+import express from 'express'
+import { ValidationResult, BlobRef } from '@atproto/lexicon'
+import { lexicons } from '../../../../lexicons'
+import { isObj, hasProp } from '../../../../util'
+import { CID } from 'multiformats/cid'
+import { HandlerAuth } from '@atproto/xrpc-server'
+
+export interface QueryParams {}
+
+export interface InputSchema {
+  did: string
+  force?: boolean
+  [k: string]: unknown
+}
+
+export interface HandlerInput {
+  encoding: 'application/json'
+  body: InputSchema
+}
+
+export interface HandlerError {
+  status: number
+  message?: string
+}
+
+export type HandlerOutput = HandlerError | void
+export type HandlerReqCtx<HA extends HandlerAuth = never> = {
+  auth: HA
+  params: QueryParams
+  input: HandlerInput
+  req: express.Request
+  res: express.Response
+}
+export type Handler<HA extends HandlerAuth = never> = (
+  ctx: HandlerReqCtx<HA>,
+) => Promise<HandlerOutput> | HandlerOutput

package/src/services/index.ts

@@ -14,6 +14,7 @@ import { LocalService } from './local'
 export function createServices(resources: {
   repoSigningKey: crypto.Keypair
   blobstore: BlobStore
+  pdsHostname: string
   appViewAgent?: AtpAgent
   appViewDid?: string
   appViewCdnUrlPattern?: string
@@ -23,6 +24,7 @@ export function createServices(resources: {
   const {
     repoSigningKey,
     blobstore,
+    pdsHostname,
     appViewAgent,
     appViewDid,
     appViewCdnUrlPattern,
@@ -41,6 +43,7 @@ export function createServices(resources: {
     ),
     local: LocalService.creator(
       repoSigningKey,
+      pdsHostname,
       appViewAgent,
       appViewDid,
       appViewCdnUrlPattern,

package/src/services/local/index.ts

@@ -39,6 +39,7 @@ export class LocalService {
   constructor(
     public db: Database,
     public signingKey: Keypair,
+    public pdsHostname: string,
     public appviewAgent?: AtpAgent,
     public appviewDid?: string,
     public appviewCdnUrlPattern?: string,
@@ -46,6 +47,7 @@ export class LocalService {
 
   static creator(
     signingKey: Keypair,
+    pdsHostname: string,
     appviewAgent?: AtpAgent,
     appviewDid?: string,
     appviewCdnUrlPattern?: string,
@@ -54,6 +56,7 @@ export class LocalService {
       new LocalService(
         db,
         signingKey,
+        pdsHostname,
         appviewAgent,
         appviewDid,
         appviewCdnUrlPattern,
@@ -62,7 +65,7 @@ export class LocalService {
 
   getImageUrl(pattern: CommonSignedUris, did: string, cid: string) {
     if (!this.appviewCdnUrlPattern) {
-      return ''
+      return `https://${this.pdsHostname}/xrpc/${ids.ComAtprotoSyncGetBlob}?did=${did}&cid=${cid}`
     }
     return util.format(this.appviewCdnUrlPattern, pattern, did, cid)
   }