@xyo-network/http-call-witness 3.0.4

package/src/Witness/Witness.ts

@@ -0,0 +1,219 @@
+/* eslint-disable max-statements */
+import { assertEx } from '@xylabs/assert'
+import { AxiosJson } from '@xylabs/axios'
+import type { Hash } from '@xylabs/hex'
+import { URL } from '@xylabs/url'
+import { AbstractWitness } from '@xyo-network/abstract-witness'
+import { PayloadHasher } from '@xyo-network/hash'
+import { PayloadBuilder } from '@xyo-network/payload-builder'
+import type { Schema } from '@xyo-network/payload-model'
+import { isPayloadOfSchemaType } from '@xyo-network/payload-model'
+import type { AxiosError } from 'axios'
+import { Axios } from 'axios'
+import { fromByteArray } from 'base64-js'
+import fillTemplate from 'es6-dynamic-template'
+
+import { checkIpfsUrl } from '../lib/index.ts'
+import type {
+  HttpCall,
+  HttpCallBase64Result,
+  HttpCallErrorResult,
+  HttpCallJsonResult,
+  HttpCallJsonResultType,
+  HttpCallResult,
+  HttpCallXmlResult,
+  MimeTypes,
+} from '../Payload/index.ts'
+import {
+  asHttpUriCall,
+  asHttpUriTemplateCall,
+  HttpCallResultSchema,
+  HttpCallSchema,
+} from '../Payload/index.ts'
+import type { HttpCallHtmlResult } from '../Payload/types/index.ts'
+import { asHttpUriCallWitnessConfig, asHttpUriTemplateCallWitnessConfig } from './Config.ts'
+import type { HttpCallWitnessParams } from './Params.ts'
+import { HttpCallWitnessConfigSchema } from './Schema.ts'
+
+export class HttpCallWitness<TParams extends HttpCallWitnessParams = HttpCallWitnessParams> extends AbstractWitness<TParams, HttpCall, HttpCallResult> {
+  static override readonly configSchemas: Schema[] = [...super.configSchemas, HttpCallWitnessConfigSchema]
+  static override readonly defaultConfigSchema: Schema = HttpCallWitnessConfigSchema
+
+  get accept(): MimeTypes {
+    return this.config.accept ?? 'application/json'
+  }
+
+  get ipfsGateway() {
+    return this.params.ipfsGateway
+  }
+
+  get timeout() {
+    return this.config.timeout
+  }
+
+  getFullUri(call?: HttpCall): string {
+    const { uri: callUri } = asHttpUriCall(call) ?? {}
+    const {
+      uriTemplate: callUriTemplate, params: callParams, queries: callQueries,
+    } = asHttpUriTemplateCall(call) ?? {}
+    const { uri: configUri } = asHttpUriCallWitnessConfig(this.config) ?? {}
+    const {
+      uriTemplate: configUriTemplate, params: configParams, queries: configQueries,
+    } = asHttpUriTemplateCallWitnessConfig(this.config) ?? {}
+
+    const params = { ...configParams, ...callParams }
+
+    let url: URL | undefined = undefined
+
+    if (callUri) {
+      url = new URL(callUri)
+    } else if (callUriTemplate) {
+      url = new URL(fillTemplate(callUriTemplate, params))
+    } else if (configUri) {
+      url = new URL(configUri)
+    } else if (configUriTemplate) {
+      url = new URL(fillTemplate(configUriTemplate, params))
+    }
+
+    if (url) {
+      const queries = Object.entries({ ...configQueries, ...callQueries })
+      queries.map(([key, value]) => url?.searchParams.set(key, value))
+      return url.href
+    }
+
+    throw new Error('Unable to determine uri. No uri/uriTemplate specified in either the call or config.')
+  }
+
+  getHeaders(headers?: Record<string, string | undefined>): Record<string, string | undefined> {
+    return {
+      ...this.params.headers, ...this.config.headers, ...headers,
+    }
+  }
+
+  protected override async observeHandler(inPayloads: HttpCall[] = []): Promise<HttpCallResult[]> {
+    await this.started('throw')
+    try {
+      const observations = await Promise.all(
+        inPayloads.filter(isPayloadOfSchemaType(HttpCallSchema)).map(async (call) => {
+          const { verb: callVerb } = call
+          const { verb: configVerb } = this.config
+          const verb = callVerb ?? configVerb ?? 'get'
+          const uri = this.getFullUri(call)
+
+          const validatedUri = assertEx(checkIpfsUrl(uri, this.ipfsGateway), () => 'Invalid URI')
+
+          if (verb === 'get') {
+            return this.httpGet(validatedUri, (await PayloadBuilder.build(call)).$hash)
+          }
+
+          const observation: HttpCallResult = {
+            call: await PayloadHasher.hash(call),
+            schema: HttpCallResultSchema,
+          }
+          return observation
+        }),
+      )
+      return observations
+    } catch (ex) {
+      const error = ex as Error
+      console.error(`Error [${this.config.name}]: ${error.message}`)
+      console.log(error.stack)
+      throw error
+    }
+  }
+
+  // eslint-disable-next-line complexity
+  private async httpGet(url: string, call: Hash, headers?: Record<string, string | undefined>): Promise<HttpCallResult> {
+    const result: HttpCallResult = {
+      call,
+      schema: HttpCallResultSchema,
+    }
+    try {
+      switch (this.accept) {
+        case 'application/json': {
+          const axios = new AxiosJson({
+            headers: { ...this.getHeaders(headers), Accept: 'application/json' }, timeout: this.timeout, decompress: true,
+          })
+          const response = await axios.get<HttpCallJsonResultType>(url)
+          if (response.status >= 200 && response.status < 300) {
+            const jsonResult = result as HttpCallJsonResult
+            jsonResult.data = response.data
+            jsonResult.contentType = 'application/json'
+          } else {
+            const errorResult = result as HttpCallErrorResult
+            errorResult.http = { status: response.status }
+          }
+          break
+        }
+        case 'application/xml':
+        case 'text/xml': {
+          const axios = new Axios({
+            headers: { ...this.getHeaders(headers), Accept: this.accept },
+            responseType: 'arraybuffer',
+            decompress: true,
+            timeout: this.timeout,
+          })
+          const response = await axios.get(url)
+          if (response.status >= 200 && response.status < 300) {
+            const xmlResult = result as HttpCallXmlResult
+            xmlResult.data = Buffer.from(response.data, 'binary').toString('utf8')
+            xmlResult.contentType = response.headers['content-type']?.toString() ?? 'application/xml'
+          } else {
+            const errorResult = result as HttpCallErrorResult
+            errorResult.http = { status: response.status }
+          }
+          break
+        }
+        case 'text/html':{
+          const axios = new Axios({
+            headers: { ...this.getHeaders(headers), Accept: this.accept },
+            responseType: 'text',
+            decompress: true,
+            timeout: this.timeout,
+          })
+          const response = await axios.get(url)
+          if (response.status >= 200 && response.status < 300) {
+            const htmlResult = result as HttpCallHtmlResult
+            htmlResult.data = response.data
+            htmlResult.contentType = response.headers['content-type']?.toString() ?? 'text/html'
+          } else {
+            const errorResult = result as HttpCallErrorResult
+            errorResult.http = { status: response.status }
+          }
+          break
+        }
+        default: {
+          const axios = new Axios({
+            headers: this.params.headers, responseType: 'arraybuffer', timeout: this.timeout, decompress: true,
+          })
+          const response = await axios.get(url)
+          if (response.status >= 200 && response.status < 300) {
+            const jsonResult = result as HttpCallBase64Result
+            jsonResult.data = fromByteArray(Buffer.from(response.data, 'binary'))
+            jsonResult.contentType = response.headers['content-type']?.toString() ?? 'application/octet-stream'
+          } else {
+            const errorResult = result as HttpCallErrorResult
+            errorResult.http = { status: response.status }
+          }
+          break
+        }
+      }
+    } catch (ex) {
+      const axiosError = ex as AxiosError
+      if (axiosError.isAxiosError) {
+        if (axiosError?.response?.status !== undefined) {
+          result.http = result.http ?? {}
+          result.http.status = axiosError?.response?.status
+        }
+        if (axiosError?.code !== undefined) {
+          result.http = result.http ?? {}
+          result.http.code = axiosError?.code
+        }
+        return result
+      } else {
+        throw ex
+      }
+    }
+    return result
+  }
+}

package/src/Witness/index.ts

@@ -0,0 +1,4 @@
+export * from './Config.ts'
+export * from './Params.ts'
+export * from './Schema.ts'
+export * from './Witness.ts'

package/src/Witness/spec/opensea.nft-call.json

@@ -0,0 +1,41 @@
+{
+  "$schema": "https://raw.githubusercontent.com/XYOracleNetwork/sdk-xyo-client-js/main/packages/manifest/src/schema.json",
+  "nodes": [
+    {
+      "config": {
+        "name": "ApiCall",
+        "schema": "network.xyo.node.config"
+      },
+      "modules": {
+        "public": [
+          {
+            "config": {
+              "language": "javascript",
+              "name": "HttpCallWitness",
+              "queries": {
+                "limit": 1
+              },
+              "schema": "network.xyo.http.call.witness.config",
+              "uriTemplate": "https://api.opensea.io/api/v2/chain/ethereum/account/${address}/nfts"
+            }
+          },
+          {
+            "config": {
+              "language": "javascript",
+              "name": "HttpCallSentinel",
+              "schema": "network.xyo.sentinel.config",
+              "synchronous": "true",
+              "tasks": [
+                {
+                  "input": true,
+                  "mod": "HttpCallWitness"
+                }
+              ]
+            }
+          }
+        ]
+      }
+    }
+  ],
+  "schema": "network.xyo.manifest"
+}

package/src/index.ts

@@ -0,0 +1,2 @@
+export * from './Payload/index.ts'
+export * from './Witness/index.ts'

package/src/lib/checkIpfsUrl.ts

@@ -0,0 +1,44 @@
+import { assertEx } from '@xylabs/assert'
+import { URL } from '@xylabs/url'
+
+const allowIpfsIoRepair = true
+
+/**
+ * Returns the equivalent IPFS gateway URL for the supplied URL.
+ * @param urlToCheck The URL to check
+ * @returns If the supplied URL is an IPFS URL, it converts the URL to the
+ * equivalent IPFS gateway URL. Otherwise, returns the original URL.
+ */
+export const checkIpfsUrl = (urlToCheck: string, ipfsGateway?: string): string => {
+  try {
+    const url = new URL(urlToCheck)
+    let protocol = url.protocol
+    let host = url.host
+    let path = url.pathname
+    const query = url.search
+    if (protocol === 'ipfs:') {
+      protocol = 'https:'
+      host = assertEx(ipfsGateway, () => 'No ipfsGateway provided')
+      path = url.host === 'ipfs' ? `ipfs${path}` : `ipfs/${url.host}${path}`
+      const root = `${protocol}//${host}/${path}`
+      return query?.length > 0 ? `${root}?${query}` : root
+    } else if (allowIpfsIoRepair && protocol === 'https' && host === 'ipfs.io') {
+      protocol = 'https:'
+      host = assertEx(ipfsGateway, () => 'No ipfsGateway provided')
+      const pathParts = path.split('/')
+      if (pathParts[0] === 'ipfs') {
+        pathParts.shift()
+      }
+      path = pathParts.join('/')
+      const root = `${protocol}//${host}/${path}`
+      return query?.length > 0 ? `${root}?${query}` : root
+    } else {
+      return urlToCheck
+    }
+  } catch {
+    // const error = ex as Error
+    // console.error(`${error.name}:${error.message} [${urlToCheck}]`)
+    // console.log(error.stack)
+    return urlToCheck
+  }
+}

package/src/lib/index.ts

@@ -0,0 +1 @@
+export * from './checkIpfsUrl.ts'

package/src/types.d.ts

@@ -0,0 +1 @@
+declare module 'es6-dynamic-template'

package/typedoc.json

@@ -0,0 +1,5 @@
+{
+  "$schema": "https://typedoc.org/schema.json",
+  "entryPoints": ["./src/index.ts"],
+  "tsconfig": "./tsconfig.typedoc.json"
+}

package/xy.config.ts

@@ -0,0 +1,10 @@
+import type { XyTsupConfig } from '@xylabs/ts-scripts-yarn3'
+const config: XyTsupConfig = {
+  compile: {
+    browser: {},
+    neutral: { src: true },
+    node: {},
+  },
+}
+
+export default config