@atproto/bsky 0.0.7 → 0.0.9

package/dist/lexicon/lexicons.d.ts

@@ -5272,6 +5272,16 @@ export declare const schemaDict: {
                             type: string;
                             refs: string[];
                         };
+                        tags: {
+                            type: string;
+                            maxLength: number;
+                            items: {
+                                type: string;
+                                maxLength: number;
+                                maxGraphemes: number;
+                            };
+                            description: string;
+                        };
                         createdAt: {
                             type: string;
                             format: string;
@@ -6408,6 +6418,18 @@ export declare const schemaDict: {
                     };
                 };
             };
+            tag: {
+                type: string;
+                description: string;
+                required: string[];
+                properties: {
+                    tag: {
+                        type: string;
+                        maxLength: number;
+                        maxGraphemes: number;
+                    };
+                };
+            };
             byteSlice: {
                 type: string;
                 description: string;

package/dist/lexicon/types/app/bsky/feed/post.d.ts

@@ -20,6 +20,7 @@ export interface Record {
         $type: string;
         [k: string]: unknown;
     };
+    tags?: string[];
     createdAt: string;
     [k: string]: unknown;
 }

package/dist/lexicon/types/app/bsky/richtext/facet.d.ts

@@ -1,7 +1,7 @@
 import { ValidationResult } from '@atproto/lexicon';
 export interface Main {
     index: ByteSlice;
-    features: (Mention | Link | {
+    features: (Mention | Link | Tag | {
         $type: string;
         [k: string]: unknown;
     })[];
@@ -21,6 +21,12 @@ export interface Link {
 }
 export declare function isLink(v: unknown): v is Link;
 export declare function validateLink(v: unknown): ValidationResult;
+export interface Tag {
+    tag: string;
+    [k: string]: unknown;
+}
+export declare function isTag(v: unknown): v is Tag;
+export declare function validateTag(v: unknown): ValidationResult;
 export interface ByteSlice {
     byteStart: number;
     byteEnd: number;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/bsky",
-  "version": "0.0.7",
+  "version": "0.0.9",
   "license": "MIT",
   "description": "Reference implementation of app.bsky App View (Bluesky API)",
   "keywords": [
@@ -37,7 +37,7 @@
     "sharp": "^0.31.2",
     "typed-emitter": "^2.1.0",
     "uint8arrays": "3.0.0",
-    "@atproto/api": "^0.6.16",
+    "@atproto/api": "^0.6.18",
     "@atproto/common": "^0.3.0",
     "@atproto/crypto": "^0.2.2",
     "@atproto/syntax": "^0.1.1",
@@ -55,10 +55,10 @@
     "@types/qs": "^6.9.7",
     "@types/sharp": "^0.31.0",
     "axios": "^0.27.2",
-    "@atproto/api": "^0.6.16",
-    "@atproto/dev-env": "^0.2.7",
+    "@atproto/api": "^0.6.18",
+    "@atproto/dev-env": "^0.2.9",
     "@atproto/lex-cli": "^0.2.1",
-    "@atproto/pds": "^0.1.16",
+    "@atproto/pds": "^0.1.18",
     "@atproto/xrpc": "^0.3.1"
   },
   "scripts": {

package/src/auto-moderator/abyss.ts

@@ -1,5 +1,6 @@
 import axios from 'axios'
 import { CID } from 'multiformats/cid'
+import { AtUri } from '@atproto/syntax'
 import * as ui8 from 'uint8arrays'
 import { resolveBlob } from '../api/blob-resolver'
 import { retryHttp } from '../util/retry'
@@ -8,7 +9,7 @@ import { IdResolver } from '@atproto/identity'
 import { labelerLogger as log } from '../logger'
 
 export interface ImageFlagger {
-  scanImage(did: string, cid: CID): Promise<string[]>
+  scanImage(did: string, cid: CID, uri: AtUri): Promise<string[]>
 }
 
 export class Abyss implements ImageFlagger {
@@ -22,11 +23,11 @@ export class Abyss implements ImageFlagger {
     this.auth = basicAuth(this.password)
   }
 
-  async scanImage(did: string, cid: CID): Promise<string[]> {
+  async scanImage(did: string, cid: CID, uri: AtUri): Promise<string[]> {
     const start = Date.now()
     const res = await retryHttp(async () => {
       try {
-        return await this.makeReq(did, cid)
+        return await this.makeReq(did, cid, uri)
       } catch (err) {
         log.warn({ err, did, cid: cid.toString() }, 'abyss request failed')
         throw err
@@ -39,20 +40,24 @@ export class Abyss implements ImageFlagger {
     return this.parseRes(res)
   }
 
-  async makeReq(did: string, cid: CID): Promise<ScannerResp> {
+  async makeReq(did: string, cid: CID, uri: AtUri): Promise<ScannerResp> {
     const { stream, contentType } = await resolveBlob(
       did,
       cid,
       this.ctx.db,
       this.ctx.idResolver,
     )
-    const { data } = await axios.post(this.getReqUrl({ did }), stream, {
-      headers: {
-        'Content-Type': contentType,
-        authorization: this.auth,
+    const { data } = await axios.post(
+      this.getReqUrl({ did, uri: uri.toString() }),
+      stream,
+      {
+        headers: {
+          'Content-Type': contentType,
+          authorization: this.auth,
+        },
+        timeout: 10000,
       },
-      timeout: 10000,
-    })
+    )
     return data
   }
 
@@ -69,8 +74,11 @@ export class Abyss implements ImageFlagger {
     return labels
   }
 
-  getReqUrl(params: { did: string }) {
-    return `${this.endpoint}/xrpc/com.atproto.unspecced.scanBlob?did=${params.did}`
+  getReqUrl(params: { did: string; uri: string }) {
+    const search = new URLSearchParams(params)
+    return `${
+      this.endpoint
+    }/xrpc/com.atproto.unspecced.scanBlob?${search.toString()}`
   }
 }
 

package/src/auto-moderator/index.ts

@@ -18,7 +18,10 @@ import { ImageUriBuilder } from '../image/uri'
 import { ImageInvalidator } from '../image/invalidator'
 import { Abyss } from './abyss'
 import { FuzzyMatcher, TextFlagger } from './fuzzy-matcher'
-import { REASONOTHER } from '../lexicon/types/com/atproto/moderation/defs'
+import {
+  REASONOTHER,
+  REASONVIOLATION,
+} from '../lexicon/types/com/atproto/moderation/defs'
 
 export class AutoModerator {
   public pushAgent?: AtpAgent
@@ -172,7 +175,7 @@ export class AutoModerator {
   async checkImgForTakedown(uri: AtUri, recordCid: CID, imgCids: CID[]) {
     if (imgCids.length < 0) return
     const results = await Promise.all(
-      imgCids.map((cid) => this.imageFlagger?.scanImage(uri.host, cid)),
+      imgCids.map((cid) => this.imageFlagger?.scanImage(uri.host, cid, uri)),
     )
     const takedownCids: CID[] = []
     for (let i = 0; i < results.length; i++) {
@@ -207,7 +210,39 @@ export class AutoModerator {
     takedownCids: CID[],
     labels: string[],
   ) {
-    const reason = `automated takedown for labels: ${labels.join(', ')}`
+    const reportReason = `automated takedown (${labels.join(
+      ', ',
+    )}). account needs review and possibly additional action`
+    const takedownReason = `automated takedown for labels: ${labels.join(', ')}`
+    log.warn(
+      {
+        uri: uri.toString(),
+        blobCids: takedownCids,
+        labels,
+      },
+      'hard takedown of record (and blobs) based on auto-matching',
+    )
+
+    if (this.services.moderation) {
+      await this.ctx.db.transaction(async (dbTxn) => {
+        // directly/locally create report, even if we use pushAgent for the takedown. don't have acctual account credentials for pushAgent, only admin auth
+        if (!this.services.moderation) {
+          // checked above, outside the transaction
+          return
+        }
+        const modSrvc = this.services.moderation(dbTxn)
+        await modSrvc.report({
+          reportedBy: this.ctx.cfg.labelerDid,
+          reasonType: REASONVIOLATION,
+          subject: {
+            uri: uri,
+            cid: recordCid,
+          },
+          reason: reportReason,
+        })
+      })
+    }
+
     if (this.pushAgent) {
       await this.pushAgent.com.atproto.admin.takeModerationAction({
         action: 'com.atproto.admin.defs#takedown',
@@ -217,7 +252,7 @@ export class AutoModerator {
           cid: recordCid.toString(),
         },
         subjectBlobCids: takedownCids.map((c) => c.toString()),
-        reason,
+        reason: takedownReason,
         createdBy: this.ctx.cfg.labelerDid,
       })
     } else {
@@ -230,7 +265,7 @@ export class AutoModerator {
           action: 'com.atproto.admin.defs#takedown',
           subject: { uri, cid: recordCid },
           subjectBlobCids: takedownCids,
-          reason,
+          reason: takedownReason,
           createdBy: this.ctx.cfg.labelerDid,
         })
         await modSrvc.takedownRecord({

package/src/db/migrations/20230920T213858047Z-add-tags-to-post.ts

@@ -0,0 +1,9 @@
+import { Kysely } from 'kysely'
+
+export async function up(db: Kysely<unknown>): Promise<void> {
+  await db.schema.alterTable('post').addColumn('tags', 'jsonb').execute()
+}
+
+export async function down(db: Kysely<unknown>): Promise<void> {
+  await db.schema.alterTable('post').dropColumn('tags').execute()
+}

package/src/db/migrations/index.ts

@@ -28,3 +28,4 @@ export * as _20230817T195936007Z from './20230817T195936007Z-native-notification
 export * as _20230830T205507322Z from './20230830T205507322Z-suggested-feeds'
 export * as _20230904T211011773Z from './20230904T211011773Z-block-lists'
 export * as _20230906T222220386Z from './20230906T222220386Z-thread-gating'
+export * as _20230920T213858047Z from './20230920T213858047Z-add-tags-to-post'

package/src/db/tables/post.ts

@@ -12,6 +12,7 @@ export interface Post {
   replyParent: string | null
   replyParentCid: string | null
   langs: string[] | null
+  tags: string[] | null
   invalidReplyRoot: boolean | null
   violatesThreadGate: boolean | null
   createdAt: string

package/src/lexicon/lexicons.ts

@@ -5622,6 +5622,16 @@ export const schemaDict = {
               type: 'union',
               refs: ['lex:com.atproto.label.defs#selfLabels'],
             },
+            tags: {
+              type: 'array',
+              maxLength: 8,
+              items: {
+                type: 'string',
+                maxLength: 640,
+                maxGraphemes: 64,
+              },
+              description: 'Additional non-inline tags describing this post.',
+            },
             createdAt: {
               type: 'string',
               format: 'datetime',
@@ -6761,6 +6771,7 @@ export const schemaDict = {
               refs: [
                 'lex:app.bsky.richtext.facet#mention',
                 'lex:app.bsky.richtext.facet#link',
+                'lex:app.bsky.richtext.facet#tag',
               ],
             },
           },
@@ -6788,6 +6799,18 @@ export const schemaDict = {
           },
         },
       },
+      tag: {
+        type: 'object',
+        description: 'A hashtag.',
+        required: ['tag'],
+        properties: {
+          tag: {
+            type: 'string',
+            maxLength: 640,
+            maxGraphemes: 64,
+          },
+        },
+      },
       byteSlice: {
         type: 'object',
         description:

package/src/lexicon/types/app/bsky/feed/post.ts

@@ -29,6 +29,8 @@ export interface Record {
   labels?:
     | ComAtprotoLabelDefs.SelfLabels
     | { $type: string; [k: string]: unknown }
+  /** Additional non-inline tags describing this post. */
+  tags?: string[]
   createdAt: string
   [k: string]: unknown
 }

package/src/lexicon/types/app/bsky/richtext/facet.ts

@@ -8,7 +8,7 @@ import { CID } from 'multiformats/cid'
 
 export interface Main {
   index: ByteSlice
-  features: (Mention | Link | { $type: string; [k: string]: unknown })[]
+  features: (Mention | Link | Tag | { $type: string; [k: string]: unknown })[]
   [k: string]: unknown
 }
 
@@ -61,6 +61,22 @@ export function validateLink(v: unknown): ValidationResult {
   return lexicons.validate('app.bsky.richtext.facet#link', v)
 }
 
+/** A hashtag. */
+export interface Tag {
+  tag: string
+  [k: string]: unknown
+}
+
+export function isTag(v: unknown): v is Tag {
+  return (
+    isObj(v) && hasProp(v, '$type') && v.$type === 'app.bsky.richtext.facet#tag'
+  )
+}
+
+export function validateTag(v: unknown): ValidationResult {
+  return lexicons.validate('app.bsky.richtext.facet#tag', v)
+}
+
 /** A text segment. Start is inclusive, end is exclusive. Indices are for utf8-encoded strings. */
 export interface ByteSlice {
   byteStart: number

package/src/services/actor/views.ts

@@ -194,8 +194,14 @@ export class ActorViews {
               mutedByList,
               blockedBy: !!bam.blockedBy([viewer, did]),
               blocking: bam.blocking([viewer, did]) ?? undefined,
-              following: prof?.viewerFollowing || undefined,
-              followedBy: prof?.viewerFollowedBy || undefined,
+              following:
+                prof?.viewerFollowing && !bam.block([viewer, did])
+                  ? prof.viewerFollowing
+                  : undefined,
+              followedBy:
+                prof?.viewerFollowedBy && !bam.block([viewer, did])
+                  ? prof.viewerFollowedBy
+                  : undefined,
             }
           : undefined,
         labels: [...actorLabels, ...selfLabels],
@@ -314,8 +320,14 @@ export class ActorViews {
               mutedByList,
               blockedBy: !!bam.blockedBy([viewer, did]),
               blocking: bam.blocking([viewer, did]) ?? undefined,
-              following: prof?.viewerFollowing || undefined,
-              followedBy: prof?.viewerFollowedBy || undefined,
+              following:
+                prof?.viewerFollowing && !bam.block([viewer, did])
+                  ? prof.viewerFollowing
+                  : undefined,
+              followedBy:
+                prof?.viewerFollowedBy && !bam.block([viewer, did])
+                  ? prof.viewerFollowedBy
+                  : undefined,
             }
           : undefined,
         labels: [...actorLabels, ...selfLabels],

package/src/services/feed/index.ts

@@ -147,6 +147,7 @@ export class FeedService {
         'post_agg.likeCount as likeCount',
         'post_agg.repostCount as repostCount',
         'post_agg.replyCount as replyCount',
+        'post.tags as tags',
         db
           .selectFrom('repost')
           .if(!viewer, (q) => q.where(noMatch))

package/src/services/indexing/plugins/post.ts

@@ -76,6 +76,9 @@ const insertFn = async (
     langs: obj.langs?.length
       ? sql<string[]>`${JSON.stringify(obj.langs)}` // sidesteps kysely's array serialization, which is non-jsonb
       : null,
+    tags: obj.tags?.length
+      ? sql<string[]>`${JSON.stringify(obj.tags)}` // sidesteps kysely's array serialization, which is non-jsonb
+      : null,
     indexedAt: timestamp,
   }
   const [insertedPost] = await Promise.all([

package/tests/auto-moderator/takedowns.test.ts

@@ -7,6 +7,7 @@ import { TestNetwork } from '@atproto/dev-env'
 import { ImageRef, SeedClient } from '../seeds/client'
 import usersSeed from '../seeds/users'
 import { CID } from 'multiformats/cid'
+import { AtUri } from '@atproto/syntax'
 import { ImageFlagger } from '../../src/auto-moderator/abyss'
 import { ImageInvalidator } from '../../src/image/invalidator'
 import { sha256 } from '@atproto/crypto'
@@ -157,7 +158,7 @@ class TestInvalidator implements ImageInvalidator {
 }
 
 class TestFlagger implements ImageFlagger {
-  async scanImage(_did: string, cid: CID): Promise<string[]> {
+  async scanImage(_did: string, cid: CID, _uri: AtUri): Promise<string[]> {
     if (cid.equals(badCid1)) {
       return ['kill-it']
     } else if (cid.equals(badCid2)) {

package/tests/views/__snapshots__/block-lists.test.ts.snap

@@ -512,7 +512,6 @@ Object {
         "viewer": Object {
           "blockedBy": false,
           "blocking": "record(0)",
-          "following": "record(4)",
           "muted": false,
         },
       },

package/tests/views/blocks.test.ts

@@ -191,17 +191,35 @@ describe('pds views with blocking', () => {
       { actor: dan },
       { headers: await network.serviceHeaders(carol) },
     )
-    expect(resCarol.data.viewer?.blocking).toBeUndefined
+    expect(resCarol.data.viewer?.blocking).toBeUndefined()
     expect(resCarol.data.viewer?.blockedBy).toBe(true)
 
     const resDan = await agent.api.app.bsky.actor.getProfile(
       { actor: carol },
       { headers: await network.serviceHeaders(dan) },
     )
-    expect(resDan.data.viewer?.blocking).toBeDefined
+    expect(resDan.data.viewer?.blocking).toBeDefined()
     expect(resDan.data.viewer?.blockedBy).toBe(false)
   })
 
+  it('unsets viewer follow state when blocked', async () => {
+    // there are follows between carol and dan
+    const { data: profile } = await agent.api.app.bsky.actor.getProfile(
+      { actor: carol },
+      { headers: await network.serviceHeaders(dan) },
+    )
+    expect(profile.viewer?.following).toBeUndefined()
+    expect(profile.viewer?.followedBy).toBeUndefined()
+    const { data: result } = await agent.api.app.bsky.graph.getBlocks(
+      {},
+      { headers: await network.serviceHeaders(dan) },
+    )
+    const blocked = result.blocks.find((block) => block.did === carol)
+    expect(blocked).toBeDefined()
+    expect(blocked?.viewer?.following).toBeUndefined()
+    expect(blocked?.viewer?.followedBy).toBeUndefined()
+  })
+
   it('returns block status on getProfiles', async () => {
     const resCarol = await agent.api.app.bsky.actor.getProfiles(
       { actors: [alice, dan] },

package/tests/views/posts.test.ts

@@ -1,4 +1,4 @@
-import AtpAgent from '@atproto/api'
+import AtpAgent, { AppBskyFeedPost } from '@atproto/api'
 import { TestNetwork } from '@atproto/dev-env'
 import { forSnapshot, stripViewerFromPost } from '../_util'
 import { SeedClient } from '../seeds/client'
@@ -7,6 +7,7 @@ import basicSeed from '../seeds/basic'
 describe('pds posts views', () => {
   let network: TestNetwork
   let agent: AtpAgent
+  let pdsAgent: AtpAgent
   let sc: SeedClient
 
   beforeAll(async () => {
@@ -14,7 +15,7 @@ describe('pds posts views', () => {
       dbPostgresSchema: 'bsky_views_posts',
     })
     agent = network.bsky.getClient()
-    const pdsAgent = network.pds.getClient()
+    pdsAgent = network.pds.getClient()
     sc = new SeedClient(pdsAgent)
     await basicSeed(sc)
     await network.processAll()
@@ -83,4 +84,27 @@ describe('pds posts views', () => {
     ].sort()
     expect(receivedUris).toEqual(expected)
   })
+
+  it('allows for creating posts with tags', async () => {
+    const post: AppBskyFeedPost.Record = {
+      text: 'hello world',
+      tags: ['javascript', 'hehe'],
+      createdAt: new Date().toISOString(),
+    }
+
+    const { uri } = await pdsAgent.api.app.bsky.feed.post.create(
+      { repo: sc.dids.alice },
+      post,
+      sc.getHeaders(sc.dids.alice),
+    )
+
+    await network.processAll()
+    await network.bsky.processAll()
+
+    const { data } = await agent.api.app.bsky.feed.getPosts({ uris: [uri] })
+
+    expect(data.posts.length).toBe(1)
+    // @ts-ignore we know it's a post record
+    expect(data.posts[0].record.tags).toEqual(['javascript', 'hehe'])
+  })
 })

package/tests/views/suggested-follows.test.ts

@@ -11,7 +11,7 @@ describe('suggested follows', () => {
 
   beforeAll(async () => {
     network = await TestNetwork.create({
-      dbPostgresSchema: 'bsky_views_suggestions',
+      dbPostgresSchema: 'bsky_views_suggested_follows',
     })
     agent = network.bsky.getClient()
     pdsAgent = network.pds.getClient()