@seeka-labs/cli-apps 3.7.0 → 3.8.2

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/src/gbq.js

@@ -0,0 +1,31 @@
+const gbq = (module.exports = {})
+const { BigQuery } = require('@google-cloud/bigquery')
+const { logger } = require('./log')
+
+gbq.createTable = async (projectId, datasetId, tableId, schema) => {
+  const bigquery = new BigQuery({ projectId })
+
+  const [table] = await bigquery.dataset(datasetId).createTable(tableId, schema)
+
+  logger.info(`Table ${table.id} created.`)
+}
+
+gbq.patchTable = async (projectId, datasetId, tableId, schema) => {
+  const bigquery = new BigQuery({ projectId })
+
+  const res = await bigquery
+    .dataset(datasetId)
+    .table(tableId)
+    .setMetadata(schema)
+
+  logger.info('Patch result:', res)
+}
+
+gbq.tableExists = async (projectId, datasetId, tableId) => {
+  const bigquery = new BigQuery({ projectId })
+
+  const table = await bigquery.dataset(datasetId).table(tableId)
+  const res = await table.exists()
+  logger.debug('Table check:', res)
+  return res[0]
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/src/log.js

@@ -0,0 +1,37 @@
+const log = (module.exports = {})
+const log4js = require('log4js')
+const utils = require('./utils')
+
+log4js.configure({
+  appenders: {
+    err: {
+      type: 'stderr',
+      layout: {
+        type: 'pattern',
+        pattern: '%[%d %p %c%] %x{plain}%x{schema}',
+        tokens: {
+          plain: (logEvent) => logEvent.data[0],
+          schema: (logEvent) => {
+            if (!logEvent.data[1]) {
+              return ''
+            }
+            return '\n' + utils.inspector(logEvent.data[1])
+          },
+        },
+      },
+    },
+  },
+  categories: {
+    default: {
+      appenders: ['err'],
+      level: 'ERROR',
+    },
+  },
+})
+
+const logger = log4js.getLogger()
+
+// Allow log level to be changed in development
+logger.level = process.env.LOG_LEVEL || 'info'
+
+log.logger = logger

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/src/utils.js

@@ -0,0 +1,17 @@
+const util = require('util')
+const utils = (module.exports = {})
+
+// Expected format is "id": "/events/server/ecommerce/v5.1.5/schema.json"
+utils.get_table_id = (id) => {
+  const idArr = id.split('/').slice(2, -1)
+  const version = idArr.pop()
+  return idArr.join('_') + '_' + version.split('.')[0]
+}
+
+utils.inspector = (obj) =>
+  util.inspect(obj, {
+    depth: null,
+    colors: process.stdout.isTTY, // Add colours if in an interactive terminal
+    compact: false,
+    breakLength: Infinity,
+  })

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/tablediff/README.md

@@ -0,0 +1,6 @@
+Useful for QA comparison of generated tables.
+
+Review and run create_tables.sh. For example it can be useful to recreate tables in a new dataset.
+
+Once run the tablediff.sh script (again review for config), can generate actual vs expected gbq schema files (from gbq itself).
+These can then be used in combination with a diff tool to highlight any differences with the expected outputs.

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/tablediff/create_tables.sh

@@ -0,0 +1,13 @@
+#Environment dependencies 
+#PROJECT=
+#DATASET=
+#SCHEMA_PATH=???/dist/dereferenced/events/
+
+for EVENT_VERSION in `cat events.list`
+do
+  echo Starting $schema
+  cp $SCHEMA_PATH/$EVENT_VERSION/schema.json ../src
+  bin/jsbq -p $PROJECT -d $DATASET -j schema.json
+done
+
+echo Finished

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/tablediff/tablediff.sh

@@ -0,0 +1,13 @@
+#Environment dependencies
+#SOURCE_DATASET=
+#TARGET_DATASET=
+
+for TABLE in `bq ls $SOURCE_DATASET | tail +3 | cut -c 1-35| sed 's; ;;g'`
+do
+  echo $TABLE
+  # Ignore descriptions for now, as there are some non-critical changes that have occurred slowly over time
+  bq show --schema $SOURCE_DATASET.$TABLE |  python -m json.tool | sed 's; (Dereferenced);;g'  | fgrep -v '"description":' > SOURCE/$TABLE
+  bq show --schema $TARGET_DATASET.$TABLE |  python -m json.tool | sed 's; (Dereferenced);;g' | fgrep -v '"description":'  > TARGET/$TABLE
+done
+
+echo "Generated pretty schemas in SOURCE and TARGET, use diff or an IDE to review differences"

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/continueOnError/additionalProperties/expected.json

@@ -0,0 +1,23 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "address",
+        "type": "RECORD",
+        "mode": "NULLABLE",
+        "fields": [
+          {
+            "name": "street_address",
+            "type": "STRING",
+            "mode": "NULLABLE"
+          },
+          {
+            "name": "country",
+            "type": "JSON",
+            "mode": "NULLABLE"
+          }
+        ]
+      }
+    ]
+  }
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/continueOnError/additionalProperties/input.json

@@ -0,0 +1,21 @@
+{
+  "id": "http://yourdomain.com/schemas/myschema.json",
+  "description": "Complex example empty Object",
+  "type": "object",
+  "properties": {
+    "address": {
+      "type": "object",
+      "properties": {
+        "street_address": {
+          "type": "string"
+        },
+        "country": {
+          "type": "object",
+          "properties": {}
+        }
+      },
+      "additionalProperties": true
+    }
+  },
+  "additionalProperties": false
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/continueOnErrorAndPreventAdditionalObjectProperties/additionalProperties/expected.json

@@ -0,0 +1,5 @@
+{
+  "schema": {
+    "fields": []
+  }
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/continueOnErrorAndPreventAdditionalObjectProperties/additionalProperties/input.json

@@ -0,0 +1,21 @@
+{
+  "id": "http://yourdomain.com/schemas/myschema.json",
+  "description": "Complex example empty Object",
+  "type": "object",
+  "properties": {
+    "address": {
+      "type": "object",
+      "properties": {
+        "street_address": {
+          "type": "string"
+        },
+        "country": {
+          "type": "object",
+          "properties": {}
+        }
+      },
+      "additionalProperties": true
+    }
+  },
+  "additionalProperties": true
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/converter.js

@@ -0,0 +1,112 @@
+const converter = require('../../src/converter')
+const assert = require('assert')
+const fs = require('fs')
+
+describe('converter integration', () => {
+  describe('default options', () => {
+    const sampleDir = './test/integration/samples'
+    // eslint-disable-next-line mocha/no-setup-in-describe
+    const testDirs = fs.readdirSync(sampleDir)
+
+    // eslint-disable-next-line mocha/no-setup-in-describe
+    testDirs.forEach((dir) => {
+      describe(dir, () => {
+        let expected
+        let result
+
+        before(() => {
+          const inJson = require(`../../${sampleDir}/${dir}/input.json`)
+          expected = require(`../../${sampleDir}/${dir}/expected.json`)
+          result = converter.run(inJson, 'p', 't')
+        })
+
+        it('converts to big query', () => {
+          assert.deepStrictEqual(result, expected)
+        })
+      })
+    })
+  })
+
+  describe('continueOnError option', () => {
+    const sampleDir = './test/integration/continueOnError'
+    // eslint-disable-next-line mocha/no-setup-in-describe
+    const testDirs = fs.readdirSync(sampleDir)
+
+    // eslint-disable-next-line mocha/no-setup-in-describe
+    testDirs.forEach((dir) => {
+      describe(dir, () => {
+        let expected
+        let result
+
+        before(() => {
+          const inJson = require(`../../${sampleDir}/${dir}/input.json`)
+          expected = require(`../../${sampleDir}/${dir}/expected.json`)
+          const options = {
+            continueOnError: true,
+          }
+          result = converter.run(inJson, options, 'p', 't')
+        })
+
+        it('converts to big query', () => {
+          assert.deepStrictEqual(result, expected)
+        })
+      })
+    })
+  })
+
+  describe('preventAdditionalObjectProperties option', () => {
+    const sampleDir = './test/integration/preventAdditionalObjectProperties'
+    // eslint-disable-next-line mocha/no-setup-in-describe
+    const testDirs = fs.readdirSync(sampleDir)
+
+    // eslint-disable-next-line mocha/no-setup-in-describe
+    testDirs.forEach((dir) => {
+      describe(dir, () => {
+        let expected
+        let result
+
+        before(() => {
+          const inJson = require(`../../${sampleDir}/${dir}/input.json`)
+          expected = require(`../../${sampleDir}/${dir}/expected.json`)
+          const options = {
+            preventAdditionalObjectProperties: true,
+          }
+          result = converter.run(inJson, options, 'p', 't')
+        })
+
+        it('converts to big query', () => {
+          assert.deepStrictEqual(result, expected)
+        })
+      })
+    })
+  })
+
+  describe('continueOnError and preventAdditionalObjectProperties options', () => {
+    const sampleDir =
+      './test/integration/continueOnErrorAndPreventAdditionalObjectProperties'
+    // eslint-disable-next-line mocha/no-setup-in-describe
+    const testDirs = fs.readdirSync(sampleDir)
+
+    // eslint-disable-next-line mocha/no-setup-in-describe
+    testDirs.forEach((dir) => {
+      describe(dir, () => {
+        let expected
+        let result
+
+        before(() => {
+          const inJson = require(`../../${sampleDir}/${dir}/input.json`)
+          expected = require(`../../${sampleDir}/${dir}/expected.json`)
+          const options = {
+            continueOnError: true,
+            preventAdditionalObjectProperties: true,
+          }
+          result = converter.run(inJson, options, 'p', 't')
+        })
+
+        it('converts to big query', () => {
+          assert.deepStrictEqual(result, expected)
+        })
+      })
+    })
+  })
+})

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/preventAdditionalObjectProperties/additionalPropertiesFalse/expected.json

@@ -0,0 +1,16 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "first_name",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "last_name",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      }
+    ]
+  }
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/preventAdditionalObjectProperties/additionalPropertiesFalse/input.json

@@ -0,0 +1,14 @@
+{
+  "id": "http://yourdomain.com/schemas/myschema.json",
+  "description": "Example description",
+  "type": "object",
+  "properties": {
+    "first_name": {
+      "type": "string"
+    },
+    "last_name": {
+      "type": "string"
+    }
+  },
+  "additionalProperties": false
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/preventAdditionalObjectProperties/unevaluatedPropertiesFalse/expected.json

@@ -0,0 +1,16 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "first_name",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      },
+      {
+        "name": "last_name",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      }
+    ]
+  }
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/preventAdditionalObjectProperties/unevaluatedPropertiesFalse/input.json

@@ -0,0 +1,14 @@
+{
+  "id": "http://yourdomain.com/schemas/myschema.json",
+  "description": "Example description",
+  "type": "object",
+  "properties": {
+    "first_name": {
+      "type": "string"
+    },
+    "last_name": {
+      "type": "string"
+    }
+  },
+  "unevaluatedProperties": false
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/samples/allOf/expected.json

@@ -0,0 +1,23 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "names",
+        "type": "RECORD",
+        "mode": "NULLABLE",
+        "fields": [
+          {
+            "name": "first_name",
+            "type": "STRING",
+            "mode": "NULLABLE"
+          },
+          {
+            "name": "last_name",
+            "type": "STRING",
+            "mode": "NULLABLE"
+          }
+        ]
+      }
+    ]
+  }
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/samples/allOf/input.json

@@ -0,0 +1,30 @@
+{
+  "id": "http://yourdomain.com/schemas/myschema.json",
+  "description": "Example description",
+  "type": "object",
+  "properties": {
+    "names": {
+      "allOf": [
+        {
+          "type": "object",
+          "properties": {
+            "first_name": {
+              "type": "string"
+            }
+          },
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "properties": {
+            "last_name": {
+              "type": "string"
+            }
+          },
+          "additionalProperties": false
+        }
+      ]
+    }
+  },
+  "additionalProperties": false
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/samples/allOf_nested/expected.json

@@ -0,0 +1,35 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "addresses",
+        "type": "RECORD",
+        "mode": "NULLABLE",
+        "fields": [
+          {
+            "name": "address",
+            "type": "RECORD",
+            "mode": "NULLABLE",
+            "fields": [
+              {
+                "name": "street_address",
+                "type": "STRING",
+                "mode": "NULLABLE"
+              },
+              {
+                "name": "country",
+                "type": "STRING",
+                "mode": "NULLABLE"
+              },
+              {
+                "name": "town",
+                "type": "STRING",
+                "mode": "NULLABLE"
+              }
+            ]
+          }
+        ]
+      }
+    ]
+  }
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/samples/allOf_nested/input.json

@@ -0,0 +1,47 @@
+{
+  "id": "http://yourdomain.com/schemas/myschema.json",
+  "description": "Example description",
+  "type": "object",
+  "properties": {
+    "addresses": {
+      "allOf": [
+        {
+          "type": "object",
+          "properties": {
+            "address": {
+              "type": "object",
+              "properties": {
+                "street_address": {
+                  "type": "string"
+                },
+                "country": {
+                  "type": "string"
+                }
+              }
+            }
+          },
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "properties": {
+            "address": {
+              "type": "object",
+              "properties": {
+                "town": {
+                  "type": "string"
+                },
+                "country": {
+                  "type": "string"
+                }
+              },
+              "additionalProperties": false
+            }
+          },
+          "additionalProperties": false
+        }
+      ]
+    }
+  },
+  "additionalProperties": false
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/samples/anyOfMultipleTypes/expected.json

@@ -0,0 +1,11 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "first_name",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      }
+    ]
+  }
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/samples/anyOfMultipleTypes/input.json

@@ -0,0 +1,19 @@
+{
+  "id": "http://yourdomain.com/schemas/myschema.json",
+  "description": "Example description",
+  "type": "object",
+  "properties": {
+    "first_name": {
+      "anyOf": [
+        {
+          "type": "string"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    }
+  },
+  "required": ["first_name"],
+  "additionalProperties": false
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/samples/complex/expected.json

@@ -0,0 +1,23 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "address",
+        "type": "RECORD",
+        "mode": "NULLABLE",
+        "fields": [
+          {
+            "name": "street_address",
+            "type": "STRING",
+            "mode": "NULLABLE"
+          },
+          {
+            "name": "country",
+            "type": "STRING",
+            "mode": "NULLABLE"
+          }
+        ]
+      }
+    ]
+  }
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/samples/complex/input.json

@@ -0,0 +1,20 @@
+{
+  "id": "http://yourdomain.com/schemas/myschema.json",
+  "description": "Complex example",
+  "type": "object",
+  "properties": {
+    "address": {
+      "type": "object",
+      "properties": {
+        "street_address": {
+          "type": "string"
+        },
+        "country": {
+          "type": "string"
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/samples/date/expected.json

@@ -0,0 +1,11 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "name": "modified",
+        "type": "DATE",
+        "mode": "NULLABLE"
+      }
+    ]
+  }
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/samples/date/input.json

@@ -0,0 +1,12 @@
+{
+  "id": "http://yourdomain.com/schemas/myschema.json",
+  "description": "Example description",
+  "type": "object",
+  "properties": {
+    "modified": {
+      "type": "string",
+      "format": "date"
+    }
+  },
+  "additionalProperties": false
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/samples/fieldDescription/expected.json

@@ -0,0 +1,12 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "description": "the first name",
+        "name": "first_name",
+        "type": "STRING",
+        "mode": "NULLABLE"
+      }
+    ]
+  }
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/samples/fieldDescription/input.json

@@ -0,0 +1,12 @@
+{
+  "id": "http://yourdomain.com/schemas/myschema.json",
+  "description": "Example description",
+  "type": "object",
+  "properties": {
+    "first_name": {
+      "description": "the first name",
+      "type": "string"
+    }
+  },
+  "additionalProperties": false
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/test/integration/samples/json/expected.json

@@ -0,0 +1,12 @@
+{
+  "schema": {
+    "fields": [
+      {
+        "description": "Some json data without specific schema.",
+        "name": "semi_structured",
+        "type": "JSON",
+        "mode": "NULLABLE"
+      }
+    ]
+  }
+}