@xyo-network/payload-builder 3.5.2 → 3.6.0-rc.2

package/src/Builder.ts

@@ -1,96 +1,92 @@
 import { assertEx } from '@xylabs/assert'
-import type { Hash } from '@xylabs/hex'
 import type {
-  AnyObject, JsonArray, JsonObject,
-} from '@xylabs/object'
-import { isJsonObject, omitBy } from '@xylabs/object'
-import { PayloadHasher } from '@xyo-network/hash'
-import type {
-  Payload, PayloadWithMeta, WithMeta,
+  Hash,
+  Hex,
+} from '@xylabs/hex'
+import {
+  isHash, isHex, toHex,
+} from '@xylabs/hex'
+import type { AnyObject } from '@xylabs/object'
+import { ObjectHasher } from '@xyo-network/hash'
+import {
+  type Payload, StorageMetaConstants, type WithStorageMeta,
 } from '@xyo-network/payload-model'
 
-import type { WithoutMeta, WithoutSchema } from './BuilderBase.ts'
-import { PayloadBuilderBase, removeMetaAndSchema } from './BuilderBase.ts'
+import { PayloadBuilderBase } from './BuilderBase.ts'
 import type { PayloadBuilderOptions } from './Options.ts'
 
-export interface BuildOptions {
-  stamp?: boolean
-  validate?: boolean
-}
-
-const omitByPredicate = <T extends object>(prefix: string) => (_: T[keyof T], key: keyof T) => {
-  assertEx(typeof key === 'string', () => `Invalid key type [${String(key)}, ${typeof key}]`)
-  return (key as string).startsWith(prefix)
-}
-
 export class PayloadBuilder<
   T extends Payload = Payload<AnyObject>,
   O extends PayloadBuilderOptions<T> = PayloadBuilderOptions<T>,
 > extends PayloadBuilderBase<T, O> {
-  static async build<T extends Payload = Payload<AnyObject>>(payload: T, options?: BuildOptions): Promise<WithMeta<T>>
-  static async build<T extends Payload = Payload<AnyObject>>(payload: T[], options?: BuildOptions): Promise<WithMeta<T>[]>
-  static async build<T extends Payload = Payload<AnyObject>>(payload: T | T[], options: BuildOptions = {}) {
-    if (Array.isArray(payload)) {
-      return await Promise.all(payload.map(payload => this.build(payload, options)))
-    } else {
-      const { stamp = false, validate = true } = options
-      const {
-        schema, $hash: incomingDataHash, $meta: incomingMeta = {},
-      } = payload as WithMeta<T>
-
-      // check for legacy signatures
-      const { _signatures } = payload as { _signatures?: JsonArray }
-      if (_signatures && !incomingMeta.signatures) {
-        incomingMeta.signatures = _signatures
-      }
-
-      const fields = removeMetaAndSchema(payload)
-      const dataHashableFields = await PayloadBuilder.dataHashableFields(schema, fields)
-      const $hash = validate || incomingDataHash === undefined ? await PayloadHasher.hash(dataHashableFields) : incomingDataHash
-      const $meta: JsonObject = { ...incomingMeta }
-      if ($meta.timestamp === undefined && stamp) {
-        $meta.timestamp = Date.now()
-      }
-      const hashableFields: WithMeta<Payload> = {
-        ...dataHashableFields, $hash, schema,
-      }
-
-      if (Object.keys($meta).length > 0) {
-        hashableFields.$meta = $meta
-      }
-
-      return hashableFields as WithMeta<T>
+  static async addSequencedStorageMeta<T extends Payload = Payload>(payload: T, hash?: Hash, dataHash?: Hash): Promise<WithStorageMeta<T>> {
+    assertEx(hash === undefined || isHash(hash), () => 'Invalid hash')
+    assertEx(dataHash === undefined || isHash(dataHash), () => 'Invalid dataHash')
+    const _hash = hash ?? await PayloadBuilder.hash(payload)
+    return {
+      ...payload,
+      _sequence: this.buildSequence(Date.now(), _hash.slice(-(StorageMetaConstants.nonceBytes * 2)) as Hex),
+      _dataHash: dataHash ?? await PayloadBuilder.dataHash(payload),
+      _hash,
     }
   }
 
-  static async dataHash<T extends Payload>(payload: T, options?: BuildOptions): Promise<Hash> {
-    return (await this.build(payload, options)).$hash
+  static async addStorageMeta<T extends Payload>(payload: T): Promise<WithStorageMeta<T>>
+  static async addStorageMeta<T extends Payload>(payloads: T[]): Promise<WithStorageMeta<T>[]>
+  static async addStorageMeta<T extends Payload>(payloads: T | T[]): Promise<WithStorageMeta<T>[] | WithStorageMeta<T>> {
+    return Array.isArray(payloads)
+      ? await (async () => {
+        const pairs = await PayloadBuilder.hashPairs(payloads)
+        return await Promise.all(pairs.map(async ([payload, hash]) => await this.addSequencedStorageMeta(
+          payload,
+          hash,
+        )))
+      })()
+      : this.addSequencedStorageMeta(
+          payloads,
+        )
+  }
+
+  static buildSequence(epoch: number, nonce: Hex): Hex {
+    assertEx(
+      epoch <= StorageMetaConstants.maxEpoch,
+      () => `epoch must be less than or equal to ${StorageMetaConstants.maxEpoch} [${epoch}]`,
+    )
+    assertEx(isHex(nonce), () => 'nonce must be a Hex type')
+    assertEx(
+      nonce.length === StorageMetaConstants.nonceBytes * 2,
+      () => `nonce must be ${StorageMetaConstants.nonceBytes} bytes [${nonce.length}] <- Hex String Length`,
+    )
+    return `${toHex(epoch, { byteSize: 4 })}${nonce}` as Hex
   }
 
-  static async dataHashPairs<T extends Payload>(payloads: T[], options?: BuildOptions): Promise<[WithMeta<T>, Hash][]> {
+  static async dataHash<T extends Payload>(payload: T): Promise<Hash> {
+    return await ObjectHasher.hash(this.omitMeta(payload))
+  }
+
+  static async dataHashPairs<T extends Payload>(payloads: T[]): Promise<[T, Hash][]> {
     return await Promise.all(
       payloads.map(async (payload) => {
-        const built = await PayloadBuilder.build(payload, options)
-        return [built, built.$hash]
+        const dataHash = await this.dataHash(payload)
+        return [payload, dataHash]
       }),
     )
   }
 
-  static async dataHashes(payloads: undefined, options?: BuildOptions): Promise<undefined>
-  static async dataHashes<T extends Payload>(payloads: T[], options?: BuildOptions): Promise<Hash[]>
-  static async dataHashes<T extends Payload>(payloads?: T[], options?: BuildOptions): Promise<Hash[] | undefined> {
+  static async dataHashes(payloads: undefined): Promise<undefined>
+  static async dataHashes<T extends Payload>(payloads: T[]): Promise<Hash[]>
+  static async dataHashes<T extends Payload>(payloads?: T[]): Promise<Hash[] | undefined> {
     return payloads
       ? await Promise.all(
         payloads.map(async (payload) => {
-          const built = await PayloadBuilder.build(payload, options)
-          return built.$hash
+          return await PayloadBuilder.dataHash(payload)
         }),
       )
       : undefined
   }
 
   static async filterExclude<T extends Payload>(payloads: T[] = [], hash: Hash[] | Hash): Promise<T[]> {
-    return await PayloadHasher.filterExcludeByHash(await this.filterExcludeByDataHash(payloads, hash), hash)
+    return await ObjectHasher.filterExcludeByHash(await this.filterExcludeByDataHash(payloads, hash), hash)
   }
 
   static async filterExcludeByDataHash<T extends Payload>(payloads: T[] = [], hash: Hash[] | Hash): Promise<T[]> {
@@ -107,8 +103,8 @@ export class PayloadBuilder<
     return (await this.dataHashPairs(payloads)).find(([_, objHash]) => objHash === hash)?.[0]
   }
 
-  static async hash<T extends Payload>(payload: T, options?: BuildOptions): Promise<Hash> {
-    return await PayloadHasher.hash(await PayloadBuilder.build(payload, options))
+  static async hash<T extends Payload>(payload: T): Promise<Hash> {
+    return await ObjectHasher.hash(this.omitStorageMeta(payload))
   }
 
   /**
@@ -116,68 +112,47 @@ export class PayloadBuilder<
    * @param objs Any array of payloads
    * @returns An array of payload/hash tuples
    */
-  static async hashPairs<T extends Payload>(payloads: T[], options?: BuildOptions): Promise<[WithMeta<T>, Hash][]> {
+  static async hashPairs<T extends Payload>(payloads: T[]): Promise<[T, Hash][]> {
     return await Promise.all(
-      payloads.map<Promise<[WithMeta<T>, Hash]>>(async (payload) => {
-        const built = await PayloadBuilder.build(payload, options)
-        return [built, await PayloadBuilder.hash(built)]
+      payloads.map<Promise<[T, Hash]>>(async (payload) => {
+        return [payload, await PayloadBuilder.hash(payload)]
       }),
     )
   }
 
-  static async hashableFields<T extends Payload = Payload<AnyObject>>(
-    schema: string,
-    fields?: WithoutSchema<WithoutMeta<T>>,
-    $meta?: JsonObject,
-    $hash?: Hash,
-    timestamp?: number,
-    stamp = false,
-  ): Promise<WithMeta<T>> {
-    const dataFields = await this.dataHashableFields<T>(schema, fields)
-    assertEx($meta === undefined || isJsonObject($meta), () => '$meta must be JsonObject')
-    const result: WithMeta<T> = omitBy(
-      {
-        ...dataFields,
-        $hash: $hash ?? (await PayloadBuilder.dataHash(dataFields)),
-        schema,
-      } as WithMeta<T>,
-      omitByPredicate('_'),
-    ) as WithMeta<T>
-
-    const clonedMeta = { ...$meta }
-
-    if (timestamp) {
-      clonedMeta.timestamp = timestamp
-    }
-
-    if (clonedMeta.timestamp === undefined && stamp) {
-      clonedMeta.timestamp = Date.now()
-    }
-
-    if (Object.keys(clonedMeta).length > 0) {
-      result.$meta = clonedMeta
-    }
-
-    return result
+  static hashableFields<T extends Payload>(
+    payload: T,
+  ): T {
+    return this.omitStorageMeta(payload)
   }
 
   static async hashes(payloads: undefined): Promise<undefined>
   static async hashes<T extends Payload>(payloads: T[]): Promise<Hash[]>
   static async hashes<T extends Payload>(payloads?: T[]): Promise<Hash[] | undefined> {
-    return await PayloadHasher.hashes(payloads)
+    return await ObjectHasher.hashes(payloads)
   }
 
-  static async toAllHashMap<T extends Payload>(objs: T[]): Promise<Record<Hash, WithMeta<T>>> {
-    const result: Record<Hash, WithMeta<T>> = {}
-    for (const pair of await this.hashPairs(objs)) {
+  static sortByStorageMeta<T extends Payload>(payloads: WithStorageMeta<T>[], direction: -1 | 1 = 1) {
+    return payloads.sort((a, b) =>
+      a._sequence < b._sequence
+        ? -direction
+        : a._sequence > b._sequence
+          ? direction
+          : 0)
+  }
+
+  static async toAllHashMap<T extends Payload>(payloads: T[]): Promise<Record<Hash, T>> {
+    const result: Record<Hash, T> = {}
+    for (const pair of await this.hashPairs(payloads)) {
+      const dataHash = await this.dataHash(pair[0])
       result[pair[1]] = pair[0]
-      result[pair[0].$hash] = pair[0]
+      result[dataHash] = pair[0]
     }
     return result
   }
 
-  static async toDataHashMap<T extends Payload>(objs: T[]): Promise<Record<Hash, WithMeta<T>>> {
-    const result: Record<Hash, WithMeta<T>> = {}
+  static async toDataHashMap<T extends Payload>(objs: T[]): Promise<Record<Hash, T>> {
+    const result: Record<Hash, T> = {}
     for (const pair of await this.dataHashPairs(objs)) {
       result[pair[1]] = pair[0]
     }
@@ -189,41 +164,18 @@ export class PayloadBuilder<
    * @param objs Any array of payloads
    * @returns A map of hashes to payloads
    */
-  static async toHashMap<T extends Payload>(objs: T[]): Promise<Record<Hash, WithMeta<T>>> {
-    const result: Record<Hash, WithMeta<T>> = {}
+  static async toHashMap<T extends Payload>(objs: T[]): Promise<Record<Hash, T>> {
+    const result: Record<Hash, T> = {}
     for (const pair of await this.hashPairs(objs)) {
       result[pair[1]] = pair[0]
     }
     return result
   }
 
-  static withoutMeta(payload: undefined): undefined
-  static withoutMeta<T extends PayloadWithMeta>(payload: T): Omit<T, '$meta'>
-  static withoutMeta<T extends PayloadWithMeta>(payloads: T[]): Omit<T, '$meta'>[]
-  static withoutMeta<T extends PayloadWithMeta>(payloads: T | T[]) {
-    if (Array.isArray(payloads)) {
-      return payloads.map(payload => this.withoutMeta(payload))
-    } else {
-      if (payloads) {
-        // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        const { $meta, ...result } = payloads
-        return result as Omit<T, '$meta'>
-      }
-    }
-  }
-
-  async build(options?: BuildOptions): Promise<WithMeta<T>> {
-    const dataHashableFields = await this.dataHashableFields()
-    return await PayloadBuilder.build<T>({
-      ...dataHashableFields, $meta: this._$meta, schema: this._schema,
-    } as Payload as T, options)
-  }
-
-  async hashableFields() {
-    return await PayloadBuilder.hashableFields(
-      assertEx(this._schema, () => 'Payload: Missing Schema'),
-      this._fields,
-      this._$meta,
-    )
+  build(): T {
+    return {
+      schema: this._schema,
+      ...this._fields,
+    } as T
   }
 }

package/src/BuilderBase.ts

@@ -1,98 +1,93 @@
 import { assertEx } from '@xylabs/assert'
-import type { Hash } from '@xylabs/hex'
-import type { AnyObject, JsonObject } from '@xylabs/object'
+import type { AnyObject } from '@xylabs/object'
 import {
   isJsonObject, omitBy, toJson,
 } from '@xylabs/object'
 import type { Promisable } from '@xylabs/promise'
 import { removeEmptyFields } from '@xyo-network/hash'
-import type {
-  Payload, Schema, WithMeta, WithOptionalMeta,
-} from '@xyo-network/payload-model'
+import type { Payload, Schema } from '@xyo-network/payload-model'
 
+import { PayloadBuilder } from './Builder.ts'
 import type { PayloadBuilderOptions } from './Options.ts'
 
 export type WithOptionalSchema<T extends Payload> = Omit<T, 'schema'> & Partial<T>
 
 export type WithoutSchema<T extends WithOptionalSchema<Payload>> = Omit<T, 'schema'>
 
-export type WithoutMeta<T extends WithOptionalMeta<Payload>> = Omit<T, '$hash' | '$meta'>
-
-export const removeMetaAndSchema = <T extends Payload>(payload: WithOptionalSchema<WithOptionalMeta<T>>): WithoutSchema<WithoutMeta<T>> => {
-  const { ...result } = payload
-  delete result.$hash
-  delete result.$meta
+export const removeMetaAndSchema = <T extends Payload>(payload: Partial<WithOptionalSchema<T>>): WithoutSchema<T> => {
+  const { ...result } = PayloadBuilder.omitMeta(payload as T) as WithOptionalSchema<T>
   delete result.schema
   return result as Omit<T, 'schema'>
 }
 
-const omitByPredicate = (prefix: string) => (_: unknown, key: string) => {
+const omitByPrefixPredicate = (prefix: string) => (_: unknown, key: string) => {
   assertEx(typeof key === 'string', () => `Invalid key type [${key}, ${typeof key}]`)
   return key.startsWith(prefix)
 }
 
 export class PayloadBuilderBase<T extends Payload = Payload<AnyObject>, O extends PayloadBuilderOptions<T> = PayloadBuilderOptions<T>> {
-  protected _$meta?: JsonObject
-  protected _fields?: WithoutSchema<WithoutMeta<T>>
+  protected _fields?: Partial<WithoutSchema<T>>
   protected _schema: Schema
 
   constructor(readonly options: O) {
-    const {
-      schema, fields, meta,
-    } = options
+    const { schema, fields } = options
     this._schema = schema
-    this._fields = removeEmptyFields(fields ?? {}) as WithoutSchema<WithoutMeta<T>>
-    this._$meta = meta
+    this._fields = removeMetaAndSchema(removeEmptyFields(structuredClone(fields ?? {})))
   }
 
-  static dataHashableFields<T extends Payload = Payload<AnyObject>>(
-    schema: string,
-    fields?: WithoutSchema<WithoutMeta<T>>,
-  ): Promisable<Omit<T, '$hash' | '$meta'>> {
-    const cleanFields = fields ? removeEmptyFields(fields) : undefined
+  static dataHashableFields<T extends Payload>(
+    schema: Schema,
+    payload: WithoutSchema<T>,
+
+  ): Promisable<Payload> {
+    const cleanFields = removeEmptyFields({ ...payload, schema })
     assertEx(
       cleanFields === undefined || isJsonObject(cleanFields),
       () => `Fields must be JsonObject: ${JSON.stringify(toJson(cleanFields), null, 2)}`,
     )
-    return omitBy(omitBy({ schema, ...cleanFields }, omitByPredicate('$')), omitByPredicate('_')) as unknown as T
+    return this.omitMeta(cleanFields) as T
   }
 
-  protected static metaFields(dataHash: Hash, otherMeta?: JsonObject, stamp = true): Promisable<JsonObject> {
-    const meta: JsonObject = { ...otherMeta }
-
-    if (!meta.timestamp && stamp) {
-      meta.timestamp = meta.timestamp ?? Date.now()
-    }
+  static omitClientMeta<T extends Payload>(payload: T, maxDepth?: number): T
+  static omitClientMeta<T extends Payload>(payloads: T[], maxDepth?: number): T[]
+  static omitClientMeta<T extends Payload>(payloads: T | T[], maxDepth = 100): T | T[] {
+    return Array.isArray(payloads)
+      ? payloads.map(payload => this.omitClientMeta(payload, maxDepth)) as T[]
+      : omitBy(payloads, omitByPrefixPredicate('$'), maxDepth) as T
+  }
 
-    return meta
+  static omitMeta<T extends Payload>(payload: T, maxDepth?: number): T
+  static omitMeta<T extends Payload>(payloads: T[], maxDepth?: number): T[]
+  static omitMeta<T extends Payload>(payloads: T | T[], maxDepth = 100): T | T[] {
+    return Array.isArray(payloads)
+      ? this.omitStorageMeta(this.omitClientMeta(payloads, maxDepth), maxDepth)
+      : this.omitStorageMeta(this.omitClientMeta(payloads, maxDepth), maxDepth)
   }
 
-  $meta(meta?: JsonObject) {
-    this._$meta = meta ?? (this._fields as WithMeta<T>).$meta
-    return this
+  static omitStorageMeta<T extends Payload>(payload: T, maxDepth?: number): T
+  static omitStorageMeta<T extends Payload>(payloads: T[], maxDepth?: number): T[]
+  static omitStorageMeta<T extends Payload>(payloads: T | T[], maxDepth = 100): T | T[] {
+    return Array.isArray(payloads)
+      ? payloads.map(payload => this.omitStorageMeta(payload, maxDepth)) as T[]
+      : omitBy(payloads, omitByPrefixPredicate('_'), maxDepth) as T
   }
 
   async dataHashableFields() {
     return await PayloadBuilderBase.dataHashableFields(
       assertEx(this._schema, () => 'Payload: Missing Schema'),
-      this._fields,
+      // TDOD: Add verification that required fields are present
+      this._fields as T,
     )
   }
 
-  // we do not require sending in $hash since it will be generated anyway
-  fields(fields: WithOptionalSchema<WithOptionalMeta<T>>) {
+  fields(fields: WithOptionalSchema<T>) {
     if (fields) {
-      const {
-        // eslint-disable-next-line @typescript-eslint/no-unused-vars
-        $meta, $hash, schema, ...fieldsOnly
-      } = fields
-      if ($meta) {
-        this.$meta($meta)
-      }
+      const fieldsClone = structuredClone(fields)
+      const { schema } = fieldsClone
       if (schema) {
         this.schema(schema)
       }
-      this._fields = removeMetaAndSchema<T>(fields)
+      this._fields = removeEmptyFields(removeMetaAndSchema<T>(fieldsClone))
     }
     return this
   }
@@ -100,8 +95,4 @@ export class PayloadBuilderBase<T extends Payload = Payload<AnyObject>, O extend
   schema(value: Schema) {
     this._schema = value
   }
-
-  protected async metaFields(dataHash: Hash, stamp = true): Promise<JsonObject> {
-    return await PayloadBuilderBase.metaFields(dataHash, this._$meta, stamp)
-  }
 }

package/src/Options.ts

@@ -3,7 +3,7 @@ import type { JsonObject } from '@xylabs/object'
 import type { Schema } from '@xyo-network/payload-model'
 
 export interface PayloadBuilderOptions<T> {
-  readonly fields?: Omit<T, 'schema' | '$hash' | '$meta'>
+  readonly fields?: Partial<T>
   readonly logger?: Logger
   readonly meta?: JsonObject
   readonly schema: Schema