@atproto/common-web 0.1.0

package/dist/ipld.d.ts

@@ -0,0 +1,9 @@
+import { CID } from 'multiformats/cid';
+export declare type JsonValue = boolean | number | string | null | undefined | unknown | Array<JsonValue> | {
+    [key: string]: JsonValue;
+};
+export declare type IpldValue = JsonValue | CID | Uint8Array | Array<IpldValue> | {
+    [key: string]: IpldValue;
+};
+export declare const jsonToIpld: (val: JsonValue) => IpldValue;
+export declare const ipldToJson: (val: IpldValue) => JsonValue;

package/dist/strings.d.ts

@@ -0,0 +1,2 @@
+export declare const utf8Len: (str: string) => number;
+export declare const graphemeLen: (str: string) => number;

package/dist/tid.d.ts

@@ -0,0 +1,20 @@
+export declare class TID {
+    str: string;
+    constructor(str: string);
+    static next(): TID;
+    static nextStr(): string;
+    static fromTime(timestamp: number, clockid: number): TID;
+    static fromStr(str: string): TID;
+    static newestFirst(a: TID, b: TID): number;
+    static oldestFirst(a: TID, b: TID): number;
+    static is(str: string): boolean;
+    timestamp(): number;
+    clockid(): number;
+    formatted(): string;
+    toString(): string;
+    compareTo(other: TID): number;
+    equals(other: TID): boolean;
+    newerThan(other: TID): boolean;
+    olderThan(other: TID): boolean;
+}
+export default TID;

package/dist/times.d.ts

@@ -0,0 +1,4 @@
+export declare const SECOND = 1000;
+export declare const MINUTE: number;
+export declare const HOUR: number;
+export declare const DAY: number;

package/dist/types.d.ts

@@ -0,0 +1,19 @@
+import { CID } from 'multiformats/cid';
+import { z } from 'zod';
+import { Def } from './check';
+export declare const schema: {
+    cid: z.ZodEffects<z.ZodEffects<z.ZodAny, any, any>, CID, any>;
+    bytes: z.ZodType<Uint8Array, z.ZodTypeDef, Uint8Array>;
+    string: z.ZodString;
+    array: z.ZodArray<z.ZodUnknown, "many">;
+    map: z.ZodRecord<z.ZodString, z.ZodUnknown>;
+    unknown: z.ZodUnknown;
+};
+export declare const def: {
+    cid: Def<CID>;
+    bytes: Def<Uint8Array>;
+    string: Def<string>;
+    map: Def<Record<string, unknown>>;
+    unknown: Def<unknown>;
+};
+export declare type ArrayEl<A> = A extends readonly (infer T)[] ? T : never;

package/dist/util.d.ts

@@ -0,0 +1,16 @@
+/// <reference types="node" />
+export declare const noUndefinedVals: <T>(obj: Record<string, T>) => Record<string, T>;
+export declare const wait: (ms: number) => Promise<unknown>;
+export declare const bailableWait: (ms: number) => {
+    bail: () => void;
+    wait: () => Promise<void>;
+};
+export declare const flattenUint8Arrays: (arrs: Uint8Array[]) => Uint8Array;
+export declare const streamToArray: (stream: AsyncIterable<Uint8Array>) => Promise<Uint8Array>;
+export declare const s32encode: (i: number) => string;
+export declare const s32decode: (s: string) => number;
+export declare const asyncFilter: <T>(arr: T[], fn: (t: T) => Promise<boolean>) => Promise<T[]>;
+export declare const isErrnoException: (err: unknown) => err is NodeJS.ErrnoException;
+export declare const errHasMsg: (err: unknown, msg: string) => boolean;
+export declare const chunkArray: <T>(arr: T[], chunkSize: number) => T[][];
+export declare const range: (num: number) => number[];

package/jest.config.js

@@ -0,0 +1,6 @@
+const base = require('../../jest.config.base.js')
+
+module.exports = {
+  ...base,
+  displayName: 'Common Web',
+}

package/package.json

@@ -0,0 +1,26 @@
+{
+  "name": "@atproto/common-web",
+  "version": "0.1.0",
+  "main": "dist/index.js",
+  "license": "MIT",
+  "scripts": {
+    "test": "jest",
+    "prettier": "prettier --check src/",
+    "prettier:fix": "prettier --write src/",
+    "lint": "eslint . --ext .ts,.tsx",
+    "lint:fix": "yarn lint --fix",
+    "verify": "run-p prettier lint",
+    "verify:fix": "yarn prettier:fix && yarn lint:fix",
+    "build": "node ./build.js",
+    "postbuild": "tsc --build tsconfig.build.json",
+    "update-main-to-dist": "node ./update-pkg.js --update-main-to-dist",
+    "update-main-to-src": "node ./update-pkg.js --update-main-to-src",
+    "prepublish": "npm run update-main-to-dist",
+    "postpublish": "npm run update-main-to-src"
+  },
+  "dependencies": {
+    "multiformats": "^9.6.4",
+    "uint8arrays": "3.0.0",
+    "zod": "^3.14.2"
+  }
+}

package/src/async.ts

@@ -0,0 +1,126 @@
+import { bailableWait } from './util'
+
+// reads values from a generator into a list
+// breaks when isDone signals `true` AND `waitFor` completes OR when a max length is reached
+// NOTE: does not signal generator to close. it *will* continue to produce values
+export const readFromGenerator = async <T>(
+  gen: AsyncGenerator<T>,
+  isDone: (last?: T) => Promise<boolean> | boolean,
+  waitFor: Promise<unknown> = Promise.resolve(),
+  maxLength = Number.MAX_SAFE_INTEGER,
+): Promise<T[]> => {
+  const evts: T[] = []
+  let bail: undefined | (() => void)
+  let hasBroke = false
+  const awaitDone = async () => {
+    if (await isDone(evts.at(-1))) {
+      return true
+    }
+    const bailable = bailableWait(20)
+    await bailable.wait()
+    bail = bailable.bail
+    if (hasBroke) return false
+    return await awaitDone()
+  }
+  const breakOn: Promise<void> = new Promise((resolve) => {
+    waitFor.then(() => {
+      awaitDone().then(() => resolve())
+    })
+  })
+
+  try {
+    while (evts.length < maxLength) {
+      const maybeEvt = await Promise.race([gen.next(), breakOn])
+      if (!maybeEvt) break
+      const evt = maybeEvt as IteratorResult<T>
+      if (evt.done) break
+      evts.push(evt.value)
+    }
+  } finally {
+    hasBroke = true
+    bail && bail()
+  }
+  return evts
+}
+
+export type Deferrable = {
+  resolve: () => void
+  complete: Promise<void>
+}
+
+export const createDeferrable = (): Deferrable => {
+  let resolve
+  const promise: Promise<void> = new Promise((res) => {
+    resolve = () => res()
+  })
+  return { resolve, complete: promise }
+}
+
+export const createDeferrables = (count: number): Deferrable[] => {
+  const list: Deferrable[] = []
+  for (let i = 0; i < count; i++) {
+    list.push(createDeferrable())
+  }
+  return list
+}
+
+export const allComplete = async (deferrables: Deferrable[]): Promise<void> => {
+  await Promise.all(deferrables.map((d) => d.complete))
+}
+
+export class AsyncBuffer<T> {
+  private buffer: T[] = []
+  private promise: Promise<void>
+  private resolve: () => void
+
+  constructor(public maxSize?: number) {
+    // Initializing to satisfy types/build, immediately reset by resetPromise()
+    this.promise = Promise.resolve()
+    this.resolve = () => null
+    this.resetPromise()
+  }
+
+  get curr(): T[] {
+    return this.buffer
+  }
+
+  get size(): number {
+    return this.buffer.length
+  }
+
+  resetPromise() {
+    this.promise = new Promise<void>((r) => (this.resolve = r))
+  }
+
+  push(item: T) {
+    this.buffer.push(item)
+    this.resolve()
+  }
+
+  pushMany(items: T[]) {
+    items.forEach((i) => this.buffer.push(i))
+    this.resolve()
+  }
+
+  async *events(): AsyncGenerator<T> {
+    while (true) {
+      await this.promise
+      if (this.maxSize && this.size > this.maxSize) {
+        throw new AsyncBufferFullError(this.maxSize)
+      }
+      const [first, ...rest] = this.buffer
+      if (first) {
+        this.buffer = rest
+        yield first
+      } else {
+        this.resetPromise()
+      }
+    }
+  }
+}
+
+export class AsyncBufferFullError extends Error {
+  constructor(maxSize: number) {
+    super(`ReachedMaxBufferSize: ${maxSize}`)
+  }
+}

package/src/check.ts

@@ -0,0 +1,25 @@
+import { ZodError } from 'zod'
+
+export interface Checkable<T> {
+  parse: (obj: unknown) => T
+  safeParse: (
+    obj: unknown,
+  ) => { success: true; data: T } | { success: false; error: ZodError }
+}
+
+export interface Def<T> {
+  name: string
+  schema: Checkable<T>
+}
+
+export const is = <T>(obj: unknown, def: Checkable<T>): obj is T => {
+  return def.safeParse(obj).success
+}
+
+export const assure = <T>(def: Checkable<T>, obj: unknown): T => {
+  return def.parse(obj)
+}
+
+export const isObject = (obj: unknown): obj is Record<string, unknown> => {
+  return typeof obj === 'object' && obj !== null
+}

package/src/index.ts

@@ -0,0 +1,10 @@
+export * as check from './check'
+export * as util from './util'
+
+export * from './async'
+export * from './util'
+export * from './tid'
+export * from './ipld'
+export * from './types'
+export * from './times'
+export * from './strings'

package/src/ipld.ts

@@ -0,0 +1,77 @@
+import { CID } from 'multiformats/cid'
+import * as ui8 from 'uint8arrays'
+
+export type JsonValue =
+  | boolean
+  | number
+  | string
+  | null
+  | undefined
+  | unknown
+  | Array<JsonValue>
+  | { [key: string]: JsonValue }
+
+export type IpldValue =
+  | JsonValue
+  | CID
+  | Uint8Array
+  | Array<IpldValue>
+  | { [key: string]: IpldValue }
+
+// @NOTE avoiding use of check.is() here only because it makes
+// these implementations slow, and they often live in hot paths.
+
+export const jsonToIpld = (val: JsonValue): IpldValue => {
+  // walk arrays
+  if (Array.isArray(val)) {
+    return val.map((item) => jsonToIpld(item))
+  }
+  // objects
+  if (val && typeof val === 'object') {
+    // check for dag json values
+    if (typeof val['$link'] === 'string' && Object.keys(val).length === 1) {
+      return CID.parse(val['$link'])
+    }
+    if (typeof val['$bytes'] === 'string' && Object.keys(val).length === 1) {
+      return ui8.fromString(val['$bytes'], 'base64')
+    }
+    // walk plain objects
+    const toReturn = {}
+    for (const key of Object.keys(val)) {
+      toReturn[key] = jsonToIpld(val[key])
+    }
+    return toReturn
+  }
+  // pass through
+  return val
+}
+
+export const ipldToJson = (val: IpldValue): JsonValue => {
+  // walk arrays
+  if (Array.isArray(val)) {
+    return val.map((item) => ipldToJson(item))
+  }
+  // objects
+  if (val && typeof val === 'object') {
+    // convert bytes
+    if (val instanceof Uint8Array) {
+      return {
+        $bytes: ui8.toString(val, 'base64'),
+      }
+    }
+    // convert cids
+    if (CID.asCID(val)) {
+      return {
+        $link: (val as CID).toString(),
+      }
+    }
+    // walk plain objects
+    const toReturn = {}
+    for (const key of Object.keys(val)) {
+      toReturn[key] = ipldToJson(val[key])
+    }
+    return toReturn
+  }
+  // pass through
+  return val as JsonValue
+}

package/src/strings.ts

@@ -0,0 +1,9 @@
+// counts the number of bytes in a utf8 string
+export const utf8Len = (str: string): number => {
+  return new TextEncoder().encode(str).byteLength
+}
+
+// counts the number of graphemes (user-displayed characters) in a string
+export const graphemeLen = (str: string): number => {
+  return [...new Intl.Segmenter().segment(str)].length
+}

package/src/tid.ts

@@ -0,0 +1,108 @@
+import { s32encode, s32decode } from './util'
+let lastTimestamp = 0
+let timestampCount = 0
+let clockid: number | null = null
+
+export class TID {
+  str: string
+
+  constructor(str: string) {
+    const noDashes = str.replace(/-/g, '')
+    if (noDashes.length !== 13) {
+      throw new Error(`Poorly formatted TID: ${noDashes.length} length`)
+    }
+    this.str = noDashes
+  }
+
+  static next(): TID {
+    // javascript does not have microsecond precision
+    // instead, we append a counter to the timestamp to indicate if multiple timestamps were created within the same millisecond
+    // take max of current time & last timestamp to prevent tids moving backwards if system clock drifts backwards
+    const time = Math.max(Date.now(), lastTimestamp)
+    if (time === lastTimestamp) {
+      timestampCount++
+    }
+    lastTimestamp = time
+    const timestamp = time * 1000 + timestampCount
+    // the bottom 32 clock ids can be randomized & are not guaranteed to be collision resistant
+    // we use the same clockid for all tids coming from this machine
+    if (clockid === null) {
+      clockid = Math.floor(Math.random() * 32)
+    }
+    return TID.fromTime(timestamp, clockid)
+  }
+
+  static nextStr(): string {
+    return TID.next().toString()
+  }
+
+  static fromTime(timestamp: number, clockid: number): TID {
+    // base32 encode with encoding variant sort (s32)
+    const str = `${s32encode(timestamp)}${s32encode(clockid).padStart(2, '2')}`
+    return new TID(str)
+  }
+
+  static fromStr(str: string): TID {
+    return new TID(str)
+  }
+
+  static newestFirst(a: TID, b: TID): number {
+    return a.compareTo(b) * -1
+  }
+
+  static oldestFirst(a: TID, b: TID): number {
+    return a.compareTo(b)
+  }
+
+  static is(str: string): boolean {
+    try {
+      TID.fromStr(str)
+      return true
+    } catch (err) {
+      return false
+    }
+  }
+
+  timestamp(): number {
+    const substr = this.str.slice(0, 11)
+    return s32decode(substr)
+  }
+
+  clockid(): number {
+    const substr = this.str.slice(11, 13)
+    return s32decode(substr)
+  }
+
+  formatted(): string {
+    const str = this.toString()
+    return `${str.slice(0, 4)}-${str.slice(4, 7)}-${str.slice(
+      7,
+      11,
+    )}-${str.slice(11, 13)}`
+  }
+
+  toString(): string {
+    return this.str
+  }
+
+  // newer > older
+  compareTo(other: TID): number {
+    if (this.str > other.str) return 1
+    if (this.str < other.str) return -1
+    return 0
+  }
+
+  equals(other: TID): boolean {
+    return this.compareTo(other) === 0
+  }
+
+  newerThan(other: TID): boolean {
+    return this.compareTo(other) > 0
+  }
+
+  olderThan(other: TID): boolean {
+    return this.compareTo(other) < 0
+  }
+}
+
+export default TID

package/src/times.ts

@@ -0,0 +1,4 @@
+export const SECOND = 1000
+export const MINUTE = SECOND * 60
+export const HOUR = MINUTE * 60
+export const DAY = HOUR * 24

package/src/types.ts

@@ -0,0 +1,44 @@
+import { CID } from 'multiformats/cid'
+import { z } from 'zod'
+import { Def } from './check'
+
+const cidSchema = z
+  .any()
+  .refine((obj: unknown) => CID.asCID(obj) !== null, {
+    message: 'Not a CID',
+  })
+  .transform((obj: unknown) => CID.asCID(obj) as CID)
+
+export const schema = {
+  cid: cidSchema,
+  bytes: z.instanceof(Uint8Array),
+  string: z.string(),
+  array: z.array(z.unknown()),
+  map: z.record(z.string(), z.unknown()),
+  unknown: z.unknown(),
+}
+
+export const def = {
+  cid: {
+    name: 'cid',
+    schema: schema.cid,
+  } as Def<CID>,
+  bytes: {
+    name: 'bytes',
+    schema: schema.bytes,
+  } as Def<Uint8Array>,
+  string: {
+    name: 'string',
+    schema: schema.string,
+  } as Def<string>,
+  map: {
+    name: 'map',
+    schema: schema.map,
+  } as Def<Record<string, unknown>>,
+  unknown: {
+    name: 'unknown',
+    schema: schema.unknown,
+  } as Def<unknown>,
+}
+
+export type ArrayEl<A> = A extends readonly (infer T)[] ? T : never

package/src/util.ts

@@ -0,0 +1,108 @@
+export const noUndefinedVals = <T>(
+  obj: Record<string, T>,
+): Record<string, T> => {
+  Object.keys(obj).forEach((k) => {
+    if (obj[k] === undefined) {
+      delete obj[k]
+    }
+  })
+  return obj
+}
+
+export const wait = (ms: number) => {
+  return new Promise((res) => setTimeout(res, ms))
+}
+
+export const bailableWait = (
+  ms: number,
+): { bail: () => void; wait: () => Promise<void> } => {
+  let bail
+  const waitPromise = new Promise<void>((res) => {
+    const timeout = setTimeout(res, ms)
+    bail = () => {
+      clearTimeout(timeout)
+      res()
+    }
+  })
+  return { bail, wait: () => waitPromise }
+}
+
+export const flattenUint8Arrays = (arrs: Uint8Array[]): Uint8Array => {
+  const length = arrs.reduce((acc, cur) => {
+    return acc + cur.length
+  }, 0)
+  const flattened = new Uint8Array(length)
+  let offset = 0
+  arrs.forEach((arr) => {
+    flattened.set(arr, offset)
+    offset += arr.length
+  })
+  return flattened
+}
+
+export const streamToArray = async (
+  stream: AsyncIterable<Uint8Array>,
+): Promise<Uint8Array> => {
+  const arrays: Uint8Array[] = []
+  for await (const chunk of stream) {
+    arrays.push(chunk)
+  }
+  return flattenUint8Arrays(arrays)
+}
+
+const S32_CHAR = '234567abcdefghijklmnopqrstuvwxyz'
+
+export const s32encode = (i: number): string => {
+  let s = ''
+  while (i) {
+    const c = i % 32
+    i = Math.floor(i / 32)
+    s = S32_CHAR.charAt(c) + s
+  }
+  return s
+}
+
+export const s32decode = (s: string): number => {
+  let i = 0
+  for (const c of s) {
+    i = i * 32 + S32_CHAR.indexOf(c)
+  }
+  return i
+}
+
+export const asyncFilter = async <T>(
+  arr: T[],
+  fn: (t: T) => Promise<boolean>,
+) => {
+  const results = await Promise.all(arr.map((t) => fn(t)))
+  return arr.filter((_, i) => results[i])
+}
+
+export const isErrnoException = (
+  err: unknown,
+): err is NodeJS.ErrnoException => {
+  return !!err && err['code']
+}
+
+export const errHasMsg = (err: unknown, msg: string): boolean => {
+  return !!err && typeof err === 'object' && err['message'] === msg
+}
+
+export const chunkArray = <T>(arr: T[], chunkSize: number): T[][] => {
+  return arr.reduce((acc, cur, i) => {
+    const chunkI = Math.floor(i / chunkSize)
+    if (!acc[chunkI]) {
+      acc[chunkI] = []
+    }
+    acc[chunkI].push(cur)
+    return acc
+  }, [] as T[][])
+}
+
+export const range = (num: number): number[] => {
+  const nums: number[] = []
+  for (let i = 0; i < num; i++) {
+    nums.push(i)
+  }
+  return nums
+}

package/tests/strings.test.ts

@@ -0,0 +1,30 @@
+import { graphemeLen, utf8Len } from '../src'
+
+describe('string', () => {
+  it('calculates utf8 string length', () => {
+    expect(utf8Len('a')).toBe(1)
+    expect(utf8Len('~')).toBe(1)
+    expect(utf8Len('ö')).toBe(2)
+    expect(utf8Len('ñ')).toBe(2)
+    expect(utf8Len('©')).toBe(2)
+    expect(utf8Len('⽘')).toBe(3)
+    expect(utf8Len('☎')).toBe(3)
+    expect(utf8Len('𓋓')).toBe(4)
+    expect(utf8Len('😀')).toBe(4)
+    expect(utf8Len('👨‍👩‍👧‍👧')).toBe(25)
+  })
+
+  it('caluclates grapheme length', () => {
+    expect(graphemeLen('a')).toBe(1)
+    expect(graphemeLen('~')).toBe(1)
+    expect(graphemeLen('ö')).toBe(1)
+    expect(graphemeLen('ñ')).toBe(1)
+    expect(graphemeLen('©')).toBe(1)
+    expect(graphemeLen('⽘')).toBe(1)
+    expect(graphemeLen('☎')).toBe(1)
+    expect(graphemeLen('𓋓')).toBe(1)
+    expect(graphemeLen('😀')).toBe(1)
+    expect(graphemeLen('👨‍👩‍👧‍👧')).toBe(1)
+    expect(graphemeLen('a~öñ©⽘☎𓋓😀👨‍👩‍👧‍👧')).toBe(10)
+  })
+})

package/tests/tid.test.ts

@@ -0,0 +1,18 @@
+import TID from '../src/tid'
+
+describe('TIDs', () => {
+  it('creates a new TID', () => {
+    const tid = TID.next()
+    const str = tid.toString()
+    expect(typeof str).toEqual('string')
+    expect(str.length).toEqual(13)
+  })
+
+  it('parses a TID', () => {
+    const tid = TID.next()
+    const str = tid.toString()
+    const parsed = TID.fromStr(str)
+    expect(parsed.timestamp()).toEqual(tid.timestamp())
+    expect(parsed.clockid()).toEqual(tid.clockid())
+  })
+})

package/tsconfig.build.json

@@ -0,0 +1,4 @@
+{
+  "extends": "./tsconfig.json",
+  "exclude": ["**/*.spec.ts", "**/*.test.ts"]
+}