@xyo-network/os-runtime 3.0.0

package/src/lib/PayloadStore.ts

@@ -0,0 +1,98 @@
+import type { ArchivistInstance, ArchivistModuleEventData } from '@xyo-network/archivist'
+import type { Payload, WithMeta, WithSources } from '@xyo-network/payload-model'
+
+import type { ExternalStore } from './ExternalStore.ts'
+import type { Listener } from './Listener.ts'
+
+export interface PayloadStoreConfig<T extends Payload = Payload> {
+  archivist: ArchivistInstance
+  getLatest: () => Promise<WithMeta<WithSources<T>>[]>
+  idFunction: (schema: Payload) => boolean
+}
+
+export class PayloadStore<T extends Payload = Payload> implements ExternalStore {
+  latest: WithMeta<T>[] = []
+  private deleteListeners: WeakRef<(args: ArchivistModuleEventData['deleted']) => void>[] = []
+  private insertListeners: WeakRef<(args: ArchivistModuleEventData['inserted']) => void>[] = []
+  private listeners: Listener[] = []
+
+  private constructor(private archivist: ArchivistInstance) {}
+
+  /**
+   * Initialize listeners on the archivist that update class member variables when new payloads are inserted
+   */
+  static async create<T extends Payload = Payload>({ archivist, getLatest, idFunction }: PayloadStoreConfig<T>) {
+    const instance = new this<T>(archivist)
+
+    const insertListener = async ({ payloads }: ArchivistModuleEventData['inserted']) => {
+      // if any of the newly inserted payloads pass the idFunction check, rerun the query
+      if (payloads.some(idFunction)) {
+        instance.latest = await getLatest()
+        instance.emitChange()
+      }
+    }
+
+    // when payloads are deleted, rerun the diviner query
+    const deleteListener = async () => {
+      instance.latest = await getLatest()
+      instance.emitChange()
+    }
+
+    const weakRefInsertListener = new WeakRef(insertListener)
+    const weakRefDeleteListener = new WeakRef(deleteListener)
+    const insertListenerRef = weakRefInsertListener.deref()
+    const deleteListenerRef = weakRefDeleteListener.deref()
+
+    if (insertListenerRef) instance.archivist.on('inserted', insertListenerRef)
+    if (deleteListenerRef) instance.archivist.on('deleted', deleteListenerRef)
+
+    instance.insertListeners.push(weakRefInsertListener)
+    instance.deleteListeners.push(weakRefDeleteListener)
+
+    instance.latest = await getLatest()
+    return instance
+  }
+
+  /**
+   * Remove all listeners and reset their class members
+   */
+  cleanupListeners() {
+    this.removeListeners(this.insertListeners, 'inserted')
+    this.removeListeners(this.deleteListeners, 'deleted')
+    this.insertListeners = []
+    this.deleteListeners = []
+  }
+
+  /**
+   * Get the current value of the store
+   */
+  getSnapshot(): WithMeta<WithSources<T>>[] {
+    return this.latest
+  }
+
+  /**
+   * Add a call back function that is invoked when new Payloads are inserted
+   *
+   * @param  {PayloadListener} onStoreChange
+   */
+  subscribe(onStoreChange: Listener) {
+    this.listeners.push(onStoreChange)
+    return () => {
+      this.listeners = this.listeners.filter(l => l !== onStoreChange)
+    }
+  }
+
+  private emitChange() {
+    for (const listener of this.listeners) {
+      listener()
+    }
+  }
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private removeListeners(listeners: WeakRef<(args: any) => void>[], eventName: string) {
+    for (const listener of listeners) {
+      const listenerRef = listener.deref()
+      if (listenerRef) this.archivist.off(eventName, listenerRef)
+    }
+  }
+}

package/src/lib/ResourceStores.ts

@@ -0,0 +1,7 @@
+import type { ExternalStore } from './ExternalStore.ts'
+
+export interface ResourceStore<Name extends string = string> {
+  cleanupListeners: () => void
+  subscriptions: Record<Name, ExternalStore['subscribe']>
+  views: Record<Name, ExternalStore['getSnapshot']>
+}

package/src/lib/index.ts

@@ -0,0 +1,10 @@
+export * from './ExternalStore.ts'
+export * from './initializeXns.ts'
+export * from './isPayload.ts'
+export * from './Listener.ts'
+export * from './ModuleAccountPaths.ts'
+export * from './ModuleNames.ts'
+export * from './NameTransforms.ts'
+export * from './PayloadStore.ts'
+export * from './ResourceStores.ts'
+export * from './tokenPlacesSplit.ts'

package/src/lib/initializeXns.ts

@@ -0,0 +1,16 @@
+import { asDivinerInstance } from '@xyo-network/diviner-model'
+import { ResolveHelper } from '@xyo-network/module-model'
+import { NameRegistrarTransformer } from '@xyo-network/module-resolver'
+import type { NodeInstance } from '@xyo-network/node-model'
+
+export const initializeXns = async (xnsNode: NodeInstance) => {
+  if (ResolveHelper.transformers.length === 0) {
+    const registrarDiviner = asDivinerInstance(await xnsNode?.resolve('AddressRecords:AddressRecordIndexDiviner'))
+    if (registrarDiviner) {
+      const transformer = new NameRegistrarTransformer(registrarDiviner, 'xyo')
+      ResolveHelper.transformers = [transformer]
+      return true
+    }
+  }
+  return false
+}

package/src/lib/isPayload.ts

@@ -0,0 +1,24 @@
+import { AsObjectFactory } from '@xylabs/object'
+import type { Payload, Schema } from '@xyo-network/payload-model'
+
+export const isObject = (x: unknown): x is Record<string | symbol | number, unknown> => {
+  return typeof x === 'object' && !Array.isArray(x)
+}
+
+export const isAnyPayload = (obj: unknown): obj is Payload => {
+  if (isObject(obj)) {
+    return typeof obj.schema === 'string'
+  }
+  return false
+}
+
+export const isPayload
+  = <T extends Payload>(schema: Schema[]) =>
+    (obj: unknown): obj is T => {
+      if (isAnyPayload(obj)) {
+        return schema.includes(obj.schema)
+      }
+      return false
+    }
+
+export const asPayload = <T extends Payload>(schema: Schema[]) => AsObjectFactory.create(isPayload<T>(schema))

package/src/lib/tokenPlacesSplit.ts

@@ -0,0 +1,17 @@
+import { assertEx } from '@xylabs/assert'
+
+export const tokenPlacesSplit = (value: bigint, places: number): [bigint, bigint] => {
+  assertEx(places >= 0, () => 'Places has to be >= 0')
+  const factor = BigInt(10 ** Math.abs(places))
+  const remainder = value % factor
+  const wholeValue = value / factor
+  return [wholeValue, remainder]
+}
+
+export const tokenPlacesSplitString = (value: bigint | undefined, places: number): string => {
+  if (value === undefined) {
+    return '-'
+  }
+  const [wholeValue, remainder] = tokenPlacesSplit(value, places)
+  return `${wholeValue}.${remainder.toString().padStart(Math.abs(places), '0')}`
+}

package/src/loadOsNode.ts

@@ -0,0 +1,43 @@
+import type { WalletInstance } from '@xyo-network/account'
+import { IndexedDbArchivist } from '@xyo-network/archivist-indexeddb'
+import { IndexedDbPayloadDiviner } from '@xyo-network/diviner-payload-indexeddb'
+import type { PackageManifestPayload } from '@xyo-network/manifest'
+import { ManifestWrapper } from '@xyo-network/manifest-wrapper'
+import { ModuleFactoryLocator } from '@xyo-network/module-factory-locator'
+import type { AttachableNodeInstance } from '@xyo-network/node-model'
+
+import { osNodeManifest } from './manifest/index.ts'
+
+// NOTE: Path "0'" is reserved for the exposed node in the kernel
+const OS_NODE_PATH = "1'"
+
+const getDefaultOsNodeLocator = (): ModuleFactoryLocator => {
+  const locator = new ModuleFactoryLocator()
+  locator.register(IndexedDbArchivist, { 'network.xyo.archivist.persistence.scope': 'device' })
+  locator.register(IndexedDbPayloadDiviner, { 'network.xyo.archivist.persistence.scope': 'device' })
+  return locator
+}
+
+export const loadOsNode = async (
+  osWallet: WalletInstance,
+  locator?: ModuleFactoryLocator,
+): Promise<[AttachableNodeInstance]> => {
+  try {
+    // Create wallets for the OS & kernel nodes
+    const osNodeWallet = await osWallet.derivePath(OS_NODE_PATH)
+
+    // Create the OS node manifest
+    const osNodeLocator: ModuleFactoryLocator = locator ?? getDefaultOsNodeLocator()
+    const manifestWrapper = new ManifestWrapper(osNodeManifest as PackageManifestPayload, osNodeWallet, osNodeLocator)
+
+    // Create the OS & kernel nodes
+    const osNode = (await manifestWrapper.loadNodes())[0]
+
+    return [osNode]
+  } catch (e) {
+    const error = e as Error
+    console.error(`Error creating os node: ${error.message}`)
+    console.error(`Error creating os node: ${error.stack}`)
+    throw new Error(`Error creating os node: ${error.message}`)
+  }
+}

package/src/manifest/ManifestReplaceableTokens.ts

@@ -0,0 +1,17 @@
+import { getXnsDomain } from '@xyo-network/kernel'
+import type { Payload } from '@xyo-network/payload-model'
+
+export const ManifestReplaceableTokens = (xnsNodeUrl: string | undefined, xnsNetwork: string | undefined) => ({
+  '[REPLACE_WITH_NS_NODE_URL]': getXnsDomain(xnsNodeUrl, xnsNetwork),
+})
+
+export const ReplaceManifestTokens = (manifest: Payload, xnsNodeUrl: string | undefined, xnsNetwork: string | undefined) => {
+  // Replace ReplaceableTokens in the manifest
+  let stringifiedManifestReplaced = JSON.stringify(manifest)
+
+  for (const [token, value] of Object.entries(ManifestReplaceableTokens(xnsNodeUrl, xnsNetwork))) {
+    stringifiedManifestReplaced = stringifiedManifestReplaced.replaceAll(token, value)
+  }
+
+  return JSON.parse(stringifiedManifestReplaced)
+}

package/src/manifest/index.ts

@@ -0,0 +1,2 @@
+export * from './ManifestReplaceableTokens.ts'
+export { default as osNodeManifest } from './os-node.manifest.json'

package/src/manifest/os-node.manifest.json

@@ -0,0 +1,132 @@
+{
+  "$schema": "https://raw.githubusercontent.com/XYOracleNetwork/sdk-xyo-client-js/main/packages/manifest/src/compilations/dapp-package-manifest-schema.json",
+  "nodes": [
+    {
+      "config": {
+        "accountPath": "0'",
+        "name": "OsNode",
+        "schema": "network.xyo.node.config"
+      },
+      "modules": {
+        "private": [],
+        "public": [
+          {
+            "config": {
+              "accountPath": "1'",
+              "name": "OSArchivist",
+              "dbName": "OSArchivist",
+              "labels": {
+                "network.xyo.archivist.persistence.scope": "device"
+              },
+              "storeName": "payloads",
+              "parents": {
+                "read": [
+                  "Archivist"
+                ]
+              },
+              "schema": "network.xyo.archivist.config"
+            }
+          },
+          {
+            "config": {
+              "accountPath": "3'",
+              "name": "DappsArchivist",
+              "dbName": "DappsArchivist",
+              "labels": {
+                "network.xyo.archivist.persistence.scope": "device"
+              },
+              "storeName": "payloads",
+              "schema": "network.xyo.archivist.config"
+            }
+          },
+          {
+            "config": {
+              "archivist": "DappsArchivist",
+              "dbVersion": 1,
+              "name": "DappsArchivistPayloadDiviner",
+              "labels": {
+                "network.xyo.archivist.persistence.scope": "device"
+              },
+              "schema": "network.xyo.diviner.payload.config"
+            }
+          },
+          {
+            "config": {
+              "accountPath": "4'",
+              "dbName": "IntentArchivist",
+              "name": "IntentArchivist",
+              "storeName": "payloads",
+              "labels": {
+                "network.xyo.archivist.persistence.scope": "device"
+              },
+              "schema": "network.xyo.archivist.config"
+            }
+          },
+          {
+            "config": {
+              "accountPath": "7",
+              "archivist": "IntentArchivist",
+              "dbVersion": 1,
+              "name": "IntentArchivistPayloadDiviner",
+              "labels": {
+                "network.xyo.archivist.persistence.scope": "device"
+              },
+              "schema": "network.xyo.diviner.payload.config",
+              "storeName": "payloads"
+            }
+          },
+          {
+            "config": {
+              "accountPath": "8",
+              "name": "RegisteredDappInterfacesArchivist",
+              "schema": "network.xyo.archivist.memory.config"
+            }
+          },
+          {
+            "config": {
+              "accountPath": "9'",
+              "name": "DappsArchivistDevelopment",
+              "parents": {
+                "write": [
+                  "DappsArchivist"
+                ]
+              },
+              "schema": "network.xyo.archivist.memory.config"
+            }
+          },
+          {
+            "config": {
+              "accountPath": "10",
+              "archivist": "DappsArchivistDevelopment",
+              "dbVersion": 1,
+              "name": "DappsArchivistPayloadDivinerDevelopment",
+              "labels": {
+                "network.xyo.generic.payload.diviner": "GenericPayloadDiviner"
+              },
+              "schema": "network.xyo.diviner.payload.generic.config"
+            }
+          },
+          {
+            "config": {
+              "name": "OsPubSubNetworkStackNode",
+              "schema": "network.xyo.node.config"
+            }
+          },
+          {
+            "config": {
+              "name": "OsSettingsStackNode",
+              "schema": "network.xyo.node.config"
+            }
+          },
+          {
+            "config": {
+              "name": "OsXyoPublicNetworkStackNode",
+              "schema": "network.xyo.node.config"
+            }
+          }
+        ]
+      }
+    }
+  ],
+  "schema": "network.xyo.manifest.package.dapp"
+}

package/src/manifests/dapp-window.manifest.json

@@ -0,0 +1,36 @@
+{
+  "$schema": "https://raw.githubusercontent.com/XYOracleNetwork/sdk-xyo-client-js/main/packages/manifest/src/compilations/dapp-package-manifest-schema.json",
+  "nodes": [
+    {
+      "config": {
+        "accountPath": "0'",
+        "name": "Dapp",
+        "schema": "network.xyo.node.config"
+      },
+      "modules": {
+        "private": [],
+        "public": [
+          {
+            "config": {
+              "accountPath": "0'/0'",
+              "name": "DappArchivist",
+              "schema": "network.xyo.archivist.memory.config"
+            }
+          },
+          {
+            "config": {
+              "accountPath": "0'/1'",
+              "archivist": "DappArchivist",
+              "labels": {
+                "network.xyo.generic.payload.diviner": "GenericPayloadDiviner"
+              },
+              "name": "DappArchivistPayloadDiviner",
+              "schema": "network.xyo.diviner.payload.config"
+            }
+          }
+        ]
+      }
+    }
+  ],
+  "schema": "network.xyo.manifest.package.dapp"
+}

package/src/manifests/index.ts

@@ -0,0 +1 @@
+export { default as DappWindowManifest } from './dapp-window.manifest.json'

package/src/profileModuleEvents.ts

@@ -0,0 +1,43 @@
+import type { Address } from '@xylabs/hex'
+import type { QuerySendStartedEventArgs } from '@xyo-network/bridge-model'
+import type { ModuleInstance } from '@xyo-network/module-model'
+import { PayloadBuilder } from '@xyo-network/payload-builder'
+
+const DISPLAY_EVENT_COUNT_FREQUENCY = 100 as const
+
+export interface ProfileEventsData {
+  allEventCount: number
+  eventCounts: { [key: string | number | symbol]: number }
+}
+
+export const profileData: Record<Address, ProfileEventsData> = {}
+
+export const profileModuleEvents = (mod: ModuleInstance) => {
+  const modRef = new WeakRef(mod) // prevent memory leak
+  mod.onAny(async (eventName, args) => {
+    const mod = modRef.deref()
+    if (mod) {
+      const data = profileData[mod.address] ?? { allEventCount: 0, eventCounts: {} }
+      profileData[mod.address] = data
+      data.allEventCount += 1
+      data.eventCounts[eventName] = (data.eventCounts[eventName] ?? 0) + 1
+      if (eventName === 'querySendStarted') {
+        const { query, payloads = [] } = args as QuerySendStartedEventArgs
+        const pairs = await PayloadBuilder.dataHashPairs(payloads)
+        const foundPair = pairs.find(([, hash]) => hash === query.query)
+        if (foundPair) {
+          const [payload] = foundPair
+          console.log(`querySendStarted: ${payload.schema}`)
+        }
+      }
+      if (eventName === 'querySendFinished') {
+        const { query } = args as QuerySendStartedEventArgs
+        console.log(`querySendFinished: ${query.query}`)
+      }
+      if (data.allEventCount % DISPLAY_EVENT_COUNT_FREQUENCY === 0) {
+        console.log(`[PROFILE] - ${mod.id} Event Counts:`)
+        console.log(data.eventCounts)
+      }
+    }
+  })
+}

package/src/stack/Base.ts

@@ -0,0 +1,153 @@
+import { assertEx } from '@xylabs/assert'
+import { forget } from '@xylabs/forget'
+import type { BaseParams } from '@xylabs/object'
+import { BaseEmitter } from '@xyo-network/module-abstract'
+import type { ModuleIdentifier } from '@xyo-network/module-model'
+import { isModuleInstance } from '@xyo-network/module-model'
+import type { AttachableNodeInstance } from '@xyo-network/node-model'
+import { asAttachableNodeInstance } from '@xyo-network/node-model'
+import type { AdapterSetCreator, NodeAdapter, NodeAdapterBaseEvents, StackBaseEvents, StackInitializer, XyOsContext } from '@xyo-network/os-model'
+
+/**
+ * Stack Base Class
+ *
+ * Listens for changes in adapter connections, update the stack node, and emits events when the stack is ready
+ */
+export class StackBase extends BaseEmitter<BaseParams, StackBaseEvents> implements StackInitializer {
+  // flag to check that all adapters are initialized
+  initialized = false
+
+  // adapters to initialize into the stack
+  private _adapterSet: NodeAdapter[]
+
+  private _context: XyOsContext
+
+  // stack node
+  private _stack: AttachableNodeInstance | undefined
+
+  // stack node module identifier for the stack
+  private _stackNodeModuleId: ModuleIdentifier
+
+  // list of initialized nodes
+  private initializedNodes: AttachableNodeInstance[] = []
+
+  // list of listeners - lister type not important since its only recalled to stop listening
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  private listeners: { adapter: NodeAdapter; eventName: string; listener: (args: any) => void }[] = []
+
+  constructor(context: XyOsContext, adapterSet: AdapterSetCreator, stackNodeModuleId: ModuleIdentifier) {
+    super({})
+    this._context = context
+    this._adapterSet = adapterSet(assertEx(this.context.kernel, 'Kernel access required'))
+    this._stackNodeModuleId = stackNodeModuleId
+  }
+
+  get adapterSet(): NodeAdapter[] {
+    return assertEx(this._adapterSet, () => `${this.stackNodeModuleId} adapters not initialized`)
+  }
+
+  get context(): XyOsContext {
+    return assertEx(this._context, () => `${this.stackNodeModuleId} stack context not initialized`)
+  }
+
+  get stack(): AttachableNodeInstance {
+    return assertEx(this._stack, () => `${this.stackNodeModuleId} stack node not initialized`)
+  }
+
+  get stackNodeModuleId(): ModuleIdentifier {
+    return assertEx(this._stackNodeModuleId, () => 'Stack node module identifier not initialized')
+  }
+
+  initialize() {
+    this.assignStack()
+    // watch for adapter connection changes
+    for (const adapter of this.adapterSet) {
+      // listen for driver ready events
+      const driverReadyListener = async ({ node }: NodeAdapterBaseEvents['driverReady']) => await this.handleDriverReady(node)
+      adapter.on('driverReady', driverReadyListener)
+      this.listeners.push({ adapter: adapter, eventName: 'driverReady', listener: driverReadyListener })
+
+      // listen for driver error events
+      const driverErrorListener = async ({ error }: NodeAdapterBaseEvents['driverError']) => {
+        console.error(`Error updating ${this.stackNodeModuleId} stack node`, error)
+        await this.emit('driverError', { error })
+      }
+      adapter.on('driverError', driverErrorListener)
+      this.listeners.push({ adapter: adapter, eventName: 'driverError', listener: driverErrorListener })
+
+      // start the adapter
+      const startAdapter = async () => await adapter.start()
+      forget(startAdapter())
+    }
+  }
+
+  /**
+   * Stops the stack
+   */
+  stop() {
+    for (const { eventName, listener, adapter } of this.listeners) {
+      // stop the adapter
+      adapter.off(eventName, listener)
+    }
+  }
+
+  /**
+   * Adds a node to the stack. Idempotent to avoid adding the
+   * same node multiple times
+   * @param node The node to add to the stack
+   * @returns
+   */
+  private async addNodeToStackNode(node: AttachableNodeInstance) {
+    // Locate the stack node
+    const stackNode = await this.getStackNode()
+
+    // If the node is already attached as there's no need to re-attach
+    const existingModule = await stackNode.resolve(node.address, { direction: 'down' })
+    if (isModuleInstance(existingModule)) return
+
+    // Register & attach the new node
+    await stackNode.register?.(node)
+    await stackNode.attach?.(node.address, true)
+  }
+
+  private assignStack() {
+    const assign = async () => {
+      const stackNode = await this.getStackNode()
+      this._stack = stackNode
+    }
+    forget(assign())
+  }
+
+  private async getStackNode() {
+    const mod = assertEx(await this.context.root.resolve(this.stackNodeModuleId), () => `${this.stackNodeModuleId} not found`)
+    return asAttachableNodeInstance(mod, () => `${this.stackNodeModuleId} not a NodeInstance`)
+  }
+
+  private async handleDriverReady(node: AttachableNodeInstance) {
+    try {
+      // eagerly emit that the driver is ready
+      await this.emit('driverReady', { node })
+
+      await this.addNodeToStackNode(node)
+
+      await this.updateInitializationState(node)
+    } catch (error) {
+      console.error(`Error updating ${this.stackNodeModuleId} stack node`, node.id, error)
+      await this.emit('stackError', { error: error as Error })
+    }
+  }
+
+  private async updateInitializationState(node: AttachableNodeInstance) {
+    // update the initialized nodes list
+    this.initializedNodes.push(node)
+
+    // update the initialized flag
+    this.initialized = this.adapterSet.every(n => n.initialized)
+
+    // notify all initialized listeners
+    if (this.initialized) {
+      await this.emit('initialized', { stack: await this.getStackNode() })
+    }
+  }
+}

package/src/stack/Manager.ts

@@ -0,0 +1,48 @@
+import type { AdapterSetCreator, StackInitializer, XyOsContext } from '@xyo-network/os-model'
+
+import type { StackBase } from './Base.ts'
+
+export interface StackConstructor {
+  new (context: XyOsContext, adapters: AdapterSetCreator): StackBase
+}
+
+export interface UninitializedStackSet {
+  adapters: AdapterSetCreator
+  stack: StackConstructor
+}
+
+export interface UninitializedStackMap {
+  [key: string]: UninitializedStackSet
+}
+
+export class StackManager {
+  private initializedStacksMap: Map<string, StackInitializer> = new Map()
+  private stackMap: UninitializedStackMap
+
+  constructor(stackMap: UninitializedStackMap) {
+    this.stackMap = stackMap
+  }
+
+  private get stacks() {
+    return this.initializedStacksMap
+  }
+
+  getStack<TStack = StackBase>(key: string) {
+    return this.initializedStacksMap.get(key) as TStack | undefined
+  }
+
+  initialize(context: XyOsContext) {
+    for (const [key, stackSet] of Object.entries(this.stackMap)) {
+      const { stack: Stack, adapters: stackNodeModuleId } = stackSet
+      const initializedStack = new Stack(context, stackNodeModuleId)
+      initializedStack.initialize()
+      this.stacks.set(key, initializedStack)
+    }
+  }
+
+  stop() {
+    for (const stack of this.stacks.values()) {
+      stack.stop()
+    }
+  }
+}