@xyo-network/payload-builder 3.5.2 → 3.6.0-rc.10

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@xyo-network/payload-builder",
-  "version": "3.5.2",
+  "version": "3.6.0-rc.10",
   "description": "Primary SDK for using XYO Protocol 2.0",
   "homepage": "https://xyo.network",
   "bugs": {
@@ -29,22 +29,23 @@
   "module": "dist/neutral/index.mjs",
   "types": "dist/neutral/index.d.ts",
   "dependencies": {
-    "@xylabs/assert": "^4.4.9",
-    "@xylabs/hex": "^4.4.9",
-    "@xylabs/logger": "^4.4.9",
-    "@xylabs/object": "^4.4.9",
-    "@xylabs/promise": "^4.4.9",
-    "@xyo-network/hash": "^3.5.2",
-    "@xyo-network/payload-model": "^3.5.2"
+    "@xylabs/assert": "^4.4.21",
+    "@xylabs/hex": "^4.4.21",
+    "@xylabs/logger": "^4.4.21",
+    "@xylabs/object": "^4.4.21",
+    "@xylabs/promise": "^4.4.21",
+    "@xyo-network/hash": "^3.6.0-rc.10",
+    "@xyo-network/payload-model": "^3.6.0-rc.10"
   },
   "devDependencies": {
-    "@xylabs/ts-scripts-yarn3": "^4.2.4",
-    "@xylabs/tsconfig": "^4.2.4",
-    "@xylabs/vitest-extended": "^4.4.9",
+    "@xylabs/ts-scripts-yarn3": "^4.2.6",
+    "@xylabs/tsconfig": "^4.2.6",
+    "@xylabs/vitest-extended": "^4.4.21",
     "typescript": "^5.7.2",
-    "vitest": "^2.1.5"
+    "vitest": "^2.1.8"
   },
   "publishConfig": {
     "access": "public"
-  }
+  },
+  "stableVersion": "3.5.2"
 }

package/src/Builder.ts

@@ -1,96 +1,93 @@
-import { assertEx } from '@xylabs/assert'
 import type { Hash } from '@xylabs/hex'
-import type {
-  AnyObject, JsonArray, JsonObject,
-} from '@xylabs/object'
-import { isJsonObject, omitBy } from '@xylabs/object'
-import { PayloadHasher } from '@xyo-network/hash'
-import type {
-  Payload, PayloadWithMeta, WithMeta,
+import type { AnyObject } from '@xylabs/object'
+import { ObjectHasher } from '@xyo-network/hash'
+import {
+  type Payload,
+  SequenceParser,
+  type WithHashMeta,
+  type WithoutStorageMeta,
+  type WithStorageMeta,
 } from '@xyo-network/payload-model'
 
-import type { WithoutMeta, WithoutSchema } from './BuilderBase.ts'
-import { PayloadBuilderBase, removeMetaAndSchema } from './BuilderBase.ts'
-import type { PayloadBuilderOptions } from './Options.ts'
-
-export interface BuildOptions {
-  stamp?: boolean
-  validate?: boolean
-}
-
-const omitByPredicate = <T extends object>(prefix: string) => (_: T[keyof T], key: keyof T) => {
-  assertEx(typeof key === 'string', () => `Invalid key type [${String(key)}, ${typeof key}]`)
-  return (key as string).startsWith(prefix)
-}
+import { PayloadBuilderBase } from './BuilderBase.ts'
 
 export class PayloadBuilder<
   T extends Payload = Payload<AnyObject>,
-  O extends PayloadBuilderOptions<T> = PayloadBuilderOptions<T>,
-> extends PayloadBuilderBase<T, O> {
-  static async build<T extends Payload = Payload<AnyObject>>(payload: T, options?: BuildOptions): Promise<WithMeta<T>>
-  static async build<T extends Payload = Payload<AnyObject>>(payload: T[], options?: BuildOptions): Promise<WithMeta<T>[]>
-  static async build<T extends Payload = Payload<AnyObject>>(payload: T | T[], options: BuildOptions = {}) {
-    if (Array.isArray(payload)) {
-      return await Promise.all(payload.map(payload => this.build(payload, options)))
+> extends PayloadBuilderBase<T> {
+  static async addHashMeta<T extends Payload>(payload: T): Promise<WithHashMeta<T>>
+  static async addHashMeta<T extends Payload>(payloads: T[]): Promise<WithHashMeta<T>[]>
+  static async addHashMeta<T extends Payload>(payloads: T | T[]): Promise<WithHashMeta<T>[] | WithHashMeta<T>> {
+    if (Array.isArray(payloads)) {
+      return await Promise.all(
+        payloads.map(async (payload) => {
+          return await this.addHashMeta(payload)
+        }),
+      )
     } else {
-      const { stamp = false, validate = true } = options
-      const {
-        schema, $hash: incomingDataHash, $meta: incomingMeta = {},
-      } = payload as WithMeta<T>
-
-      // check for legacy signatures
-      const { _signatures } = payload as { _signatures?: JsonArray }
-      if (_signatures && !incomingMeta.signatures) {
-        incomingMeta.signatures = _signatures
+      const _hash = await PayloadBuilder.hash(payloads)
+      const _dataHash = await PayloadBuilder.dataHash(payloads)
+      return {
+        ...payloads,
+        _dataHash,
+        _hash,
       }
+    }
+  }
 
-      const fields = removeMetaAndSchema(payload)
-      const dataHashableFields = await PayloadBuilder.dataHashableFields(schema, fields)
-      const $hash = validate || incomingDataHash === undefined ? await PayloadHasher.hash(dataHashableFields) : incomingDataHash
-      const $meta: JsonObject = { ...incomingMeta }
-      if ($meta.timestamp === undefined && stamp) {
-        $meta.timestamp = Date.now()
-      }
-      const hashableFields: WithMeta<Payload> = {
-        ...dataHashableFields, $hash, schema,
-      }
+  static async addSequencedStorageMeta<T extends Payload = Payload>(payload: T): Promise<WithStorageMeta<T>> {
+    const withHashMeta = await this.addHashMeta(payload)
+    const _sequence = SequenceParser.from(Date.now(), withHashMeta._hash).localSequence
+    return {
+      ...withHashMeta,
+      _sequence,
+    }
+  }
 
-      if (Object.keys($meta).length > 0) {
-        hashableFields.$meta = $meta
-      }
+  static async addStorageMeta<T extends Payload>(payload: T): Promise<WithStorageMeta<T>>
+  static async addStorageMeta<T extends Payload>(payloads: T[]): Promise<WithStorageMeta<T>[]>
+  static async addStorageMeta<T extends Payload>(payloads: T | T[]): Promise<WithStorageMeta<T>[] | WithStorageMeta<T>> {
+    return Array.isArray(payloads)
+      ? await (async () => {
+        return await Promise.all(payloads.map(async payload => await this.addSequencedStorageMeta(
+          payload,
+        )))
+      })()
+      : this.addSequencedStorageMeta(
+          payloads,
+        )
+  }
 
-      return hashableFields as WithMeta<T>
-    }
+  static compareStorageMeta(a: WithStorageMeta<Payload>, b: WithStorageMeta<Payload>) {
+    return a._sequence > b._sequence ? 1 : a._sequence < b._sequence ? -1 : 0
   }
 
-  static async dataHash<T extends Payload>(payload: T, options?: BuildOptions): Promise<Hash> {
-    return (await this.build(payload, options)).$hash
+  static async dataHash<T extends Payload>(payload: T): Promise<Hash> {
+    return await ObjectHasher.hash(this.omitMeta(payload))
   }
 
-  static async dataHashPairs<T extends Payload>(payloads: T[], options?: BuildOptions): Promise<[WithMeta<T>, Hash][]> {
+  static async dataHashPairs<T extends Payload>(payloads: T[]): Promise<[T, Hash][]> {
     return await Promise.all(
       payloads.map(async (payload) => {
-        const built = await PayloadBuilder.build(payload, options)
-        return [built, built.$hash]
+        const dataHash = await this.dataHash(payload)
+        return [payload, dataHash]
       }),
     )
   }
 
-  static async dataHashes(payloads: undefined, options?: BuildOptions): Promise<undefined>
-  static async dataHashes<T extends Payload>(payloads: T[], options?: BuildOptions): Promise<Hash[]>
-  static async dataHashes<T extends Payload>(payloads?: T[], options?: BuildOptions): Promise<Hash[] | undefined> {
+  static async dataHashes(payloads: undefined): Promise<undefined>
+  static async dataHashes<T extends Payload>(payloads: T[]): Promise<Hash[]>
+  static async dataHashes<T extends Payload>(payloads?: T[]): Promise<Hash[] | undefined> {
     return payloads
       ? await Promise.all(
         payloads.map(async (payload) => {
-          const built = await PayloadBuilder.build(payload, options)
-          return built.$hash
+          return await PayloadBuilder.dataHash(payload)
         }),
       )
       : undefined
   }
 
   static async filterExclude<T extends Payload>(payloads: T[] = [], hash: Hash[] | Hash): Promise<T[]> {
-    return await PayloadHasher.filterExcludeByHash(await this.filterExcludeByDataHash(payloads, hash), hash)
+    return await ObjectHasher.filterExcludeByHash(await this.filterExcludeByDataHash(payloads, hash), hash)
   }
 
   static async filterExcludeByDataHash<T extends Payload>(payloads: T[] = [], hash: Hash[] | Hash): Promise<T[]> {
@@ -107,8 +104,8 @@ export class PayloadBuilder<
     return (await this.dataHashPairs(payloads)).find(([_, objHash]) => objHash === hash)?.[0]
   }
 
-  static async hash<T extends Payload>(payload: T, options?: BuildOptions): Promise<Hash> {
-    return await PayloadHasher.hash(await PayloadBuilder.build(payload, options))
+  static async hash<T extends Payload>(payload: T): Promise<Hash> {
+    return await ObjectHasher.hash(this.omitStorageMeta(payload))
   }
 
   /**
@@ -116,68 +113,47 @@ export class PayloadBuilder<
    * @param objs Any array of payloads
    * @returns An array of payload/hash tuples
    */
-  static async hashPairs<T extends Payload>(payloads: T[], options?: BuildOptions): Promise<[WithMeta<T>, Hash][]> {
+  static async hashPairs<T extends Payload>(payloads: T[]): Promise<[T, Hash][]> {
     return await Promise.all(
-      payloads.map<Promise<[WithMeta<T>, Hash]>>(async (payload) => {
-        const built = await PayloadBuilder.build(payload, options)
-        return [built, await PayloadBuilder.hash(built)]
+      payloads.map<Promise<[T, Hash]>>(async (payload) => {
+        return [payload, await PayloadBuilder.hash(payload)]
       }),
     )
   }
 
-  static async hashableFields<T extends Payload = Payload<AnyObject>>(
-    schema: string,
-    fields?: WithoutSchema<WithoutMeta<T>>,
-    $meta?: JsonObject,
-    $hash?: Hash,
-    timestamp?: number,
-    stamp = false,
-  ): Promise<WithMeta<T>> {
-    const dataFields = await this.dataHashableFields<T>(schema, fields)
-    assertEx($meta === undefined || isJsonObject($meta), () => '$meta must be JsonObject')
-    const result: WithMeta<T> = omitBy(
-      {
-        ...dataFields,
-        $hash: $hash ?? (await PayloadBuilder.dataHash(dataFields)),
-        schema,
-      } as WithMeta<T>,
-      omitByPredicate('_'),
-    ) as WithMeta<T>
-
-    const clonedMeta = { ...$meta }
-
-    if (timestamp) {
-      clonedMeta.timestamp = timestamp
-    }
-
-    if (clonedMeta.timestamp === undefined && stamp) {
-      clonedMeta.timestamp = Date.now()
-    }
-
-    if (Object.keys(clonedMeta).length > 0) {
-      result.$meta = clonedMeta
-    }
-
-    return result
+  static hashableFields<T extends Payload>(
+    payload: T,
+  ): WithoutStorageMeta<T> {
+    return this.omitStorageMeta(payload)
   }
 
   static async hashes(payloads: undefined): Promise<undefined>
   static async hashes<T extends Payload>(payloads: T[]): Promise<Hash[]>
   static async hashes<T extends Payload>(payloads?: T[]): Promise<Hash[] | undefined> {
-    return await PayloadHasher.hashes(payloads)
+    return await ObjectHasher.hashes(payloads)
   }
 
-  static async toAllHashMap<T extends Payload>(objs: T[]): Promise<Record<Hash, WithMeta<T>>> {
-    const result: Record<Hash, WithMeta<T>> = {}
-    for (const pair of await this.hashPairs(objs)) {
+  static sortByStorageMeta<T extends Payload>(payloads: WithStorageMeta<T>[], direction: -1 | 1 = 1) {
+    return payloads.sort((a, b) =>
+      a._sequence < b._sequence
+        ? -direction
+        : a._sequence > b._sequence
+          ? direction
+          : 0)
+  }
+
+  static async toAllHashMap<T extends Payload>(payloads: T[]): Promise<Record<Hash, T>> {
+    const result: Record<Hash, T> = {}
+    for (const pair of await this.hashPairs(payloads)) {
+      const dataHash = await this.dataHash(pair[0])
       result[pair[1]] = pair[0]
-      result[pair[0].$hash] = pair[0]
+      result[dataHash] = pair[0]
     }
     return result
   }
 
-  static async toDataHashMap<T extends Payload>(objs: T[]): Promise<Record<Hash, WithMeta<T>>> {
-    const result: Record<Hash, WithMeta<T>> = {}
+  static async toDataHashMap<T extends Payload>(objs: T[]): Promise<Record<Hash, T>> {
+    const result: Record<Hash, T> = {}
     for (const pair of await this.dataHashPairs(objs)) {
       result[pair[1]] = pair[0]
     }
@@ -189,41 +165,19 @@ export class PayloadBuilder<
    * @param objs Any array of payloads
    * @returns A map of hashes to payloads
    */
-  static async toHashMap<T extends Payload>(objs: T[]): Promise<Record<Hash, WithMeta<T>>> {
-    const result: Record<Hash, WithMeta<T>> = {}
+  static async toHashMap<T extends Payload>(objs: T[]): Promise<Record<Hash, T>> {
+    const result: Record<Hash, T> = {}
     for (const pair of await this.hashPairs(objs)) {
       result[pair[1]] = pair[0]
     }
     return result
   }
 
-  static withoutMeta(payload: undefined): undefined
-  static withoutMeta<T extends PayloadWithMeta>(payload: T): Omit<T, '$meta'>
-  static withoutMeta<T extends PayloadWithMeta>(payloads: T[]): Omit<T, '$meta'>[]
-  static withoutMeta<T extends PayloadWithMeta>(payloads: T | T[]) {
-    if (Array.isArray(payloads)) {
-      return payloads.map(payload => this.withoutMeta(payload))
-    } else {
-      if (payloads) {
-        // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        const { $meta, ...result } = payloads
-        return result as Omit<T, '$meta'>
-      }
-    }
-  }
-
-  async build(options?: BuildOptions): Promise<WithMeta<T>> {
-    const dataHashableFields = await this.dataHashableFields()
-    return await PayloadBuilder.build<T>({
-      ...dataHashableFields, $meta: this._$meta, schema: this._schema,
-    } as Payload as T, options)
-  }
-
-  async hashableFields() {
-    return await PayloadBuilder.hashableFields(
-      assertEx(this._schema, () => 'Payload: Missing Schema'),
-      this._fields,
-      this._$meta,
-    )
+  build(): T {
+    return {
+      schema: this._schema,
+      ...this._fields,
+      ...this._meta,
+    } as T
   }
 }

package/src/BuilderBase.ts

@@ -1,107 +1,105 @@
 import { assertEx } from '@xylabs/assert'
-import type { Hash } from '@xylabs/hex'
-import type { AnyObject, JsonObject } from '@xylabs/object'
+import type { AnyObject, EmptyObject } from '@xylabs/object'
 import {
-  isJsonObject, omitBy, toJson,
+  isJsonObject, omitByPrefix, pickByPrefix, toJson,
 } from '@xylabs/object'
 import type { Promisable } from '@xylabs/promise'
 import { removeEmptyFields } from '@xyo-network/hash'
 import type {
-  Payload, Schema, WithMeta, WithOptionalMeta,
+  Payload, Schema,
+  WithOnlyClientMeta,
+  WithOptionalSchema,
+  WithoutClientMeta,
+  WithoutMeta,
+  WithoutPrivateStorageMeta,
+  WithoutSchema,
+  WithoutStorageMeta,
 } from '@xyo-network/payload-model'
 
 import type { PayloadBuilderOptions } from './Options.ts'
 
-export type WithOptionalSchema<T extends Payload> = Omit<T, 'schema'> & Partial<T>
-
-export type WithoutSchema<T extends WithOptionalSchema<Payload>> = Omit<T, 'schema'>
-
-export type WithoutMeta<T extends WithOptionalMeta<Payload>> = Omit<T, '$hash' | '$meta'>
-
-export const removeMetaAndSchema = <T extends Payload>(payload: WithOptionalSchema<WithOptionalMeta<T>>): WithoutSchema<WithoutMeta<T>> => {
-  const { ...result } = payload
-  delete result.$hash
-  delete result.$meta
+export const omitSchema = <T extends WithOptionalSchema>(payload: T): WithoutSchema<T> => {
+  const result = structuredClone(payload)
   delete result.schema
-  return result as Omit<T, 'schema'>
+  return result
 }
 
-const omitByPredicate = (prefix: string) => (_: unknown, key: string) => {
-  assertEx(typeof key === 'string', () => `Invalid key type [${key}, ${typeof key}]`)
-  return key.startsWith(prefix)
-}
-
-export class PayloadBuilderBase<T extends Payload = Payload<AnyObject>, O extends PayloadBuilderOptions<T> = PayloadBuilderOptions<T>> {
-  protected _$meta?: JsonObject
-  protected _fields?: WithoutSchema<WithoutMeta<T>>
+export class PayloadBuilderBase<T extends Payload = Payload<AnyObject>> {
+  protected _fields?: WithoutMeta<WithoutSchema<T>>
+  protected _meta?: WithOnlyClientMeta<T>
   protected _schema: Schema
 
-  constructor(readonly options: O) {
-    const {
-      schema, fields, meta,
-    } = options
+  constructor(readonly options: PayloadBuilderOptions) {
+    const { schema } = options
     this._schema = schema
-    this._fields = removeEmptyFields(fields ?? {}) as WithoutSchema<WithoutMeta<T>>
-    this._$meta = meta
   }
 
-  static dataHashableFields<T extends Payload = Payload<AnyObject>>(
-    schema: string,
-    fields?: WithoutSchema<WithoutMeta<T>>,
-  ): Promisable<Omit<T, '$hash' | '$meta'>> {
-    const cleanFields = fields ? removeEmptyFields(fields) : undefined
+  static dataHashableFields<T extends Payload>(
+    schema: Schema,
+    payload: WithoutSchema<T>,
+
+  ): Promisable<WithoutMeta<T>> {
+    const cleanFields = removeEmptyFields({ ...payload, schema })
     assertEx(
       cleanFields === undefined || isJsonObject(cleanFields),
       () => `Fields must be JsonObject: ${JSON.stringify(toJson(cleanFields), null, 2)}`,
     )
-    return omitBy(omitBy({ schema, ...cleanFields }, omitByPredicate('$')), omitByPredicate('_')) as unknown as T
+    return this.omitMeta(cleanFields) as WithoutMeta<T>
   }
 
-  protected static metaFields(dataHash: Hash, otherMeta?: JsonObject, stamp = true): Promisable<JsonObject> {
-    const meta: JsonObject = { ...otherMeta }
+  static omitClientMeta<T extends EmptyObject>(payload: T, maxDepth?: number): WithoutClientMeta<T>
+  static omitClientMeta<T extends EmptyObject>(payloads: T[], maxDepth?: number): WithoutClientMeta<T>[]
+  static omitClientMeta<T extends EmptyObject>(payloads: T | T[], maxDepth = 100): WithoutClientMeta<T> | WithoutClientMeta<T>[] {
+    return Array.isArray(payloads)
+      ? payloads.map(payload => this.omitClientMeta(payload, maxDepth))
+      : omitByPrefix(payloads, '$', maxDepth)
+  }
 
-    if (!meta.timestamp && stamp) {
-      meta.timestamp = meta.timestamp ?? Date.now()
-    }
+  static omitMeta<T extends EmptyObject>(payload: T, maxDepth?: number): WithoutMeta<T>
+  static omitMeta<T extends EmptyObject>(payloads: T[], maxDepth?: number): WithoutMeta<T>[]
+  static omitMeta<T extends EmptyObject>(payloads: T | T[], maxDepth = 100): WithoutMeta<T> | WithoutMeta<T>[] {
+    return Array.isArray(payloads)
+      ? payloads.map(payload => this.omitMeta(payload, maxDepth))
+      : this.omitStorageMeta(this.omitClientMeta(payloads, maxDepth), maxDepth) as unknown as WithoutMeta<T>
+  }
 
-    return meta
+  static omitPrivateStorageMeta<T extends EmptyObject>(payload: T, maxDepth?: number): WithoutPrivateStorageMeta<T>
+  static omitPrivateStorageMeta<T extends EmptyObject>(payloads: T[], maxDepth?: number): WithoutPrivateStorageMeta<T>[]
+  static omitPrivateStorageMeta<T extends EmptyObject>(payloads: T | T[], maxDepth = 100): WithoutPrivateStorageMeta<T> | WithoutPrivateStorageMeta<T>[] {
+    return Array.isArray(payloads)
+      ? payloads.map(payload => this.omitPrivateStorageMeta(payload, maxDepth))
+      : omitByPrefix(payloads, '__', maxDepth)
   }
 
-  $meta(meta?: JsonObject) {
-    this._$meta = meta ?? (this._fields as WithMeta<T>).$meta
-    return this
+  static omitStorageMeta<T extends EmptyObject>(payload: T, maxDepth?: number): WithoutStorageMeta<T>
+  static omitStorageMeta<T extends EmptyObject>(payloads: T[], maxDepth?: number): WithoutStorageMeta<T>[]
+  static omitStorageMeta<T extends EmptyObject>(payloads: T | T[], maxDepth = 100): WithoutStorageMeta<T> | WithoutStorageMeta<T>[] {
+    return Array.isArray(payloads)
+      ? payloads.map(payload => this.omitStorageMeta(payload, maxDepth))
+      : omitByPrefix(payloads, '_', maxDepth)
   }
 
   async dataHashableFields() {
     return await PayloadBuilderBase.dataHashableFields(
       assertEx(this._schema, () => 'Payload: Missing Schema'),
-      this._fields,
+      // TODO: Add verification that required fields are present
+      this._fields as WithoutSchema<T>,
     )
   }
 
-  // we do not require sending in $hash since it will be generated anyway
-  fields(fields: WithOptionalSchema<WithOptionalMeta<T>>) {
-    if (fields) {
-      const {
-        // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        $meta, $hash, schema, ...fieldsOnly
-      } = fields
-      if ($meta) {
-        this.$meta($meta)
-      }
-      if (schema) {
-        this.schema(schema)
-      }
-      this._fields = removeMetaAndSchema<T>(fields)
-    }
+  fields(fields: WithoutMeta<WithoutSchema<T>>) {
+    // we need to do the cast here since ts seems to not like nested, yet same, generics
+    this._fields = omitSchema(PayloadBuilderBase.omitMeta(removeEmptyFields(structuredClone(fields)))) as unknown as WithoutMeta<WithoutSchema<T>>
     return this
   }
 
-  schema(value: Schema) {
-    this._schema = value
+  meta(meta: WithOnlyClientMeta<T>) {
+    // we need to do the cast here since ts seems to not like nested, yet same, generics
+    this._meta = pickByPrefix(meta, '$') as WithOnlyClientMeta<T>
+    return this
   }
 
-  protected async metaFields(dataHash: Hash, stamp = true): Promise<JsonObject> {
-    return await PayloadBuilderBase.metaFields(dataHash, this._$meta, stamp)
+  schema(value: Schema) {
+    this._schema = value
   }
 }

package/src/Options.ts

@@ -1,10 +1,7 @@
 import type { Logger } from '@xylabs/logger'
-import type { JsonObject } from '@xylabs/object'
 import type { Schema } from '@xyo-network/payload-model'
 
-export interface PayloadBuilderOptions<T> {
-  readonly fields?: Omit<T, 'schema' | '$hash' | '$meta'>
+export interface PayloadBuilderOptions {
   readonly logger?: Logger
-  readonly meta?: JsonObject
   readonly schema: Schema
 }