@seeka-labs/cli-apps 3.7.0 → 3.8.2

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/CHANGELOG.md

@@ -0,0 +1,139 @@
+# Changelog
+
+## v7.0.0 (03/04/2023)
+
+- Updated converter to generate big numeric instead of numeric for number type.
+- Updated dependencies.
+
+## v6.1.0 (27/01/2023)
+
+- unevaluatedProperties now supported when checking for additional properties.
+
+## v6.0.0 (24/01/2023)
+
+- Objects with no properties convert to JSON fields.
+- Updated dependencies.
+
+## v5.1.0 (11/08/2022)
+
+- JSBQ CLI output now compatible with Google's bq CLI.
+- Logging code reduced.
+- Updated dependencies.
+
+## v5.0.1 (04/07/2022)
+
+- JSBQ CLI now supports draft-7+ schemas.
+- Updated dependencies.
+
+## v5.0.0 (30/12/2021)
+
+- Updated converter to generate numeric instead of float for number type.
+- Fixed field name validation.
+- Now supports TIME columns.
+- Updated to node 16, and updated dependencies.
+
+## v4.1.0 (27/09/2021)
+
+- Now checks for field name format.
+
+## v4.0.1 (12/02/2021)
+
+- Support more formats for table ID utility.
+
+## v4.0.0 (24/12/2020)
+
+- Dropped support for node 8.
+- Updated dependencies.
+
+## v3.4.0 (02/06/2020)
+
+- Date formats now supported in `OneOf` or `AnyOf`.
+- Repeated descriptions in `OneOf` or `AnyOf` are no longer duplicated.
+
+## v3.3.0 (23/05/2020)
+
+- Schema problems can now be ignored, with warnings shown.
+
+## v3.2.1 (23/05/2020)
+
+- Updated to node 12, and updated dependencies.
+
+## v3.2.0 (27/02/2020)
+
+- Now supports DATE columns.
+- Simplified custom error class.
+
+## v3.1.1 (07/08/2019)
+
+- Logging now sent to STDERR.
+
+## v3.1.0 (30/06/2019)
+
+- Now throws an error if properties of an object has not been given or does not contain any fields.
+- Object descriptions are now processed correctly
+
+## v3.0.0 (08/05/2019)
+
+- `additionalProperties` check is now optional.
+
+## v2.0.3 (07/05/2019)
+
+- Set requirePartitionFilter to true when creating GBQ tables
+
+## v2.0.2 (09/04/2019)
+
+- Supports nested combined schemas with null types.
+- `jsbq` can convert and print a schema without touching BigQuery.
+
+## v2.0.1 (07/04/2019)
+
+- Now throws an error if no type is given for a property.
+
+## v2.0.0 (01/04/2019)
+
+- Make `"additionalProperties": false` required for all `object` type properties in input JSON Schema.
+
+## v1.0.0 (30/01/2019)
+
+- Support for nested descriptions.
+
+## v0.1.2 (11/11/2018)
+
+- Improved error message when dealing with union types.
+
+## v0.1.1 (01/11/2018)
+
+- Bug fix for jsbq command.
+
+## v0.1.0 (23/10/2018)
+
+- Rewrite with improved support for `allOf`, `anyOf` and `oneOf` combined schemas.
+
+## v0.0.7 (13/08/2018)
+
+- Support fields with nullable types using `anyOf` method.
+
+## v0.0.6 (11/08/2018)
+
+- Supports `allOf`, `anyOf` and `oneOf` combined schemas.
+
+## v0.0.5 (29/07/2018)
+
+- Supports timestamp fields.
+
+## v0.0.4 (27/07/2018)
+
+- Supports repeated fields.
+
+## v0.0.3 (25/07/2018)
+
+- Now supporting fields with type(s) as an array.
+- Field descriptions now supported.
+
+## v0.0.2 (23/07/2018)
+
+- Required fields supported.
+
+## v0.0.1 (21/07/2018)
+
+- Initial release.

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2018 The Dumb Terminal
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/README.md

@@ -0,0 +1,70 @@
+# jsonschema-bigquery
+
+[![npm](https://img.shields.io/npm/v/jsonschema-bigquery.svg)](https://www.npmjs.com/package/jsonschema-bigquery)
+[![Node.js CI](https://github.com/thedumbterminal/jsonschema-bigquery/actions/workflows/main.yml/badge.svg)](https://github.com/thedumbterminal/jsonschema-bigquery/actions/workflows/main.yml)
+
+Convert JSON schema to Google BigQuery schema
+
+This includes the ability to:
+
+1. Create tables
+1. Patch tables
+
+Further enhancements are planned: delete tables (dev only), create dataset, set data ACLs
+
+Note that some features involve bespoke interpretation of schema details suited to our environment.
+
+## Install
+
+    npm install jsonschema-bigquery
+
+## Consume
+
+    jsbq -p <gbq project> -d <gbq dataset> -j <json schema file> --preventAdditionalObjectProperties --continueOnError
+
+For embedded usage the following will allow and support runtime schema conversion and table maintenance:
+
+    const jsonSchemaBigquery = require('jsonschema-bigquery')
+    const bigquerySchema = jsonSchemaBigquery.run(jsonSchemaObject, options)
+
+Please ensure that the input JSON schema is dereferenced so that all external references have been resolved. [json-schema-ref-parser](https://www.npmjs.com/package/json-schema-ref-parser) can do this, prior to using this module.
+
+### Options
+
+```
+{
+  preventAdditionalObjectProperties: true,
+  continueOnError: false
+}
+```
+
+- `preventAdditionalObjectProperties` - boolean, check for additional object properties in schemas.
+- `continueOnError` - boolean, continues conversion if problem JSON is encountered. Problems will be excluded from resulting schema.
+
+### Usage with bq CLI tool
+
+Google maintains a CLI tool called `bq` which allows the management of BigQuery tables:
+
+https://cloud.google.com/bigquery/docs/reference/bq-cli-reference
+
+To create a table from a JSON schema using other options that our `jsbq` does not support, use the following commands:
+
+First, output the generated GBQ schema to a file:
+
+```
+npx jsbq -j test/integration/samples/complex/input.json > /tmp/schema.json
+```
+
+Then run the `bq` command to create a table:
+
+```
+bq mk --schema=/tmp/schema.json test_dataset.test_table
+```
+
+Please see the bq reference for table creation options:
+
+https://cloud.google.com/bigquery/docs/reference/bq-cli-reference#bq_mk
+
+## Test
+
+    npm test

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/bin/jsbq

@@ -0,0 +1,101 @@
+#!/usr/bin/env node
+
+const jsbq = (module.exports = {})
+
+const gbq = require('../src/gbq')
+const { get_table_id } = require('../src/utils')
+const converter = require('../src/converter')
+const { logger } = require('../src/log')
+const fs = require('fs')
+const { promisify } = require('util')
+
+jsbq.process = async (project, datasetName, jsonSchema, options) => {
+  logger.info('Processing JSON schema...')
+  const tableOptions = converter.run(jsonSchema, options)
+  logger.info('Generated BigQuery schema:')
+  console.log(JSON.stringify(tableOptions.schema.fields))
+
+  if (!project && !datasetName) {
+    return logger.info('Skipping table operations')
+  }
+
+  const schemaId = jsonSchema.$id || jsonSchema.id
+  logger.info('Extracting table name from schema ID:', schemaId)
+  const tableName = get_table_id(schemaId)
+  logger.info('Table name:', tableName)
+
+  logger.info('Setting table options...')
+  tableOptions.friendly_name = jsonSchema.title
+  tableOptions.description = jsonSchema.description || jsonSchema.title
+  tableOptions.timePartitioning = {
+    type: 'DAY',
+    requirePartitionFilter: true,
+  }
+
+  logger.info('Checking if table exists...')
+  const tableExists = await gbq.tableExists(project, datasetName, tableName)
+  if (tableExists) {
+    logger.info('Patching table:', tableName)
+    await gbq.patchTable(project, datasetName, tableName, tableOptions)
+  } else {
+    logger.info('Creating table:', tableName)
+    await gbq.createTable(project, datasetName, tableName, tableOptions)
+  }
+  logger.info('Finished')
+}
+
+jsbq.run = async () => {
+  const readFile = promisify(fs.readFile)
+
+  const argv = require('yargs')
+    .usage(
+      'Usage: $0 -p [project] -d [dataset] -j <json schema file> --preventAdditionalObjectProperties --continueOnError',
+    )
+    .options({
+      j: {
+        describe: 'JSON schema file',
+        demandOption: true,
+      },
+      p: {
+        describe: 'GCP project name',
+      },
+      d: {
+        describe: 'BigQuery dataset name',
+      },
+      preventAdditionalObjectProperties: {
+        describe: 'boolean, check for additional object properties in schemas.',
+        type: 'boolean',
+        default: false,
+      },
+      continueOnError: {
+        describe:
+          'boolean, if error in json schema, skip element in big query schema and continue.',
+        type: 'boolean',
+        default: false,
+      },
+      debug: {
+        describe: 'Enable debug logging',
+        type: 'boolean',
+        default: false,
+      },
+    }).argv
+
+  if (argv.debug) {
+    logger.level = 'debug'
+    logger.debug('Debug mode enabled')
+  }
+
+  const schemaData = await readFile(argv.j)
+  const options = {
+    preventAdditionalObjectProperties: argv.preventAdditionalObjectProperties,
+    continueOnError: argv.continueOnError,
+  }
+  const jsonSchema = JSON.parse(schemaData)
+  logger.debug('Input schema: ', jsonSchema)
+  return jsbq.process(argv.p, argv.d, jsonSchema, options)
+}
+
+jsbq.run().catch((e) => {
+  logger.error(e)
+  process.exit(1)
+})

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/package.json

@@ -0,0 +1,66 @@
+{
+  "name": "jsonschema-bigquery-custom",
+  "version": "1.2.9",
+  "description": "Customised v7.0.0 NPM package https://www.npmjs.com/package/jsonschema-bigquery",
+  "main": "src/converter.js",
+  "scripts": {
+    "ci": "npm run lint && npm run coverage",
+    "test": "mocha test/unit test/integration",
+    "test:seeka": "bin/jsbq -j '/home/dan/Code/seeka-labs/app/apps/google-bigquery/app/jsonschema-bigquery-custom/test/input-jsonschema.json' > output-jsonschema.json",
+    "coverage": "nyc npm run test",
+    "lint:eslint": "eslint .",
+    "lint:prettier": "prettier -c .",
+    "lint": "npm run lint:prettier && npm run lint:eslint",
+    "format": "prettier --write ."
+  },
+  "bin": {
+    "jsbq": "./bin/jsbq"
+  },
+  "engines": {
+    "node": ">=12"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/thedumbterminal/jsonschema-bigquery.git"
+  },
+  "keywords": [
+    "json",
+    "schema",
+    "google",
+    "bigquery",
+    "convert"
+  ],
+  "author": {
+    "name": "thedumbterminal",
+    "email": "github@thedumbterminal.co.uk",
+    "url": "http://www.thedumbterminal.co.uk"
+  },
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/thedumbterminal/jsonschema-bigquery/issues"
+  },
+  "homepage": "https://github.com/thedumbterminal/jsonschema-bigquery#readme",
+  "devDependencies": {
+    "eslint": "^8.5.0",
+    "eslint-plugin-import": "^2.29.1",
+    "eslint-plugin-mocha": "^10.1.0",
+    "eslint-plugin-n": "^17.10.1",
+    "eslint-plugin-promise": "^7.0.0",
+    "mocha": "^11.0.1",
+    "nyc": "^17.0.0",
+    "prettier": "^3.3.3"
+  },
+  "dependencies": {
+    "@google-cloud/bigquery": "^8.0.0",
+    "lodash": "^4.17.20",
+    "log4js": "^6.3.0",
+    "yargs": "^17.3.1"
+  },
+  "nyc": {
+    "check-coverage": true,
+    "branches": 88,
+    "lines": 98,
+    "functions": 100,
+    "statements": 97
+  }
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/src/converter.js

@@ -0,0 +1,282 @@
+const converter = (module.exports = {})
+const _ = require('lodash')
+const { SchemaError } = require('./errors')
+const { logger } = require('./log')
+
+const JSON_SCHEMA_TO_BIGQUERY_TYPE_DICT = {
+  boolean: 'BOOLEAN',
+  'date-time': 'TIMESTAMP',
+  integer: 'INTEGER',
+  number: 'BIGNUMERIC',
+  string: 'STRING',
+  date: 'DATE',
+  time: 'TIME',
+}
+
+const OFS = ['allOf', 'anyOf', 'oneOf']
+
+const BIGQUERY_FIELD_NAME_REGEXP = /^[a-z_]([a-z0-9_]+|)$/i
+
+converter._copy = (o) => _.clone(o, false)
+converter._deepCopy = (o) => _.clone(o, true)
+
+converter._merge_property = (
+  merge_type,
+  property_name,
+  destination_value,
+  source_value,
+) => {
+  // Merges two properties.
+  if (destination_value === undefined && source_value === undefined) {
+    return undefined
+  }
+  if (destination_value === undefined) return source_value
+  if (source_value === undefined) return destination_value
+  if (
+    typeof destination_value === 'boolean' &&
+    typeof source_value === 'boolean'
+  ) {
+    return destination_value && source_value
+  }
+  if (_.isPlainObject(destination_value) && _.isPlainObject(source_value)) {
+    return converter._merge_dicts(merge_type, destination_value, source_value)
+  }
+  let destination_list
+  if (Array.isArray(destination_value)) {
+    destination_list = converter._copy(destination_value)
+  } else {
+    destination_list = [destination_value]
+  }
+  let source_list
+  if (Array.isArray(source_value)) {
+    source_list = source_value
+  } else {
+    source_list = [source_value]
+  }
+  if (property_name === 'description') {
+    destination_list.push(source_value)
+    return _.uniq(destination_list).join(' ')
+  }
+  if (property_name === 'required' && ['anyOf', 'oneOf'].includes(merge_type)) {
+    return destination_list.filter((v) => source_list.includes(v))
+  }
+  if (property_name === 'format') {
+    // if we have multiple formats we have to remove them all
+    return _.uniq(destination_list).length === 1 ? destination_list.shift() : ''
+  }
+  destination_list.push(
+    ...source_list.filter((v) => !destination_list.includes(v)),
+  )
+  return destination_list
+}
+
+/**
+ * Merges multiple sources given as an Array was *dicts,
+ *
+ * Cloned from the original merge_dicts from python-based bigjson This method had variable
+ * expansion, varags and two slightly different usages which appeared to be incompatible.
+ *
+ * Could be returned to a single method
+ *
+ * See merge_dicts for the alternate call pattern
+ */
+converter._merge_dicts_array = (merge_type, dest_dict, source_dicts) => {
+  const result = converter._deepCopy(dest_dict)
+  for (let source_dict of source_dicts) {
+    // First check if we need to recurse and merge deeper results first
+    for (const x_of of OFS) {
+      if (_.has(source_dict, x_of)) {
+        source_dict = converter._merge_dicts_array(
+          x_of,
+          source_dict,
+          source_dict[x_of],
+        )
+        delete source_dict[x_of]
+      }
+    }
+
+    const keys = Object.keys(source_dict)
+    for (const name of keys) {
+      const merged_property = converter._merge_property(
+        merge_type,
+        name,
+        result[name],
+        source_dict[name],
+      )
+      if (merged_property !== undefined) {
+        result[name] = merged_property
+      }
+    }
+  }
+  return result
+}
+
+/**
+ * Merges a single object
+ *
+ * The original merge_dicts from python-based bigjson This method had variable expansion, varags and
+ * two slightly different usages which appeared to be incompatible.
+ *
+ * Could be returned to a single method
+ *
+ * See merge_dicts_array above for the alternate call pattern
+ */
+converter._merge_dicts = (merge_type, dest_dict, source_dict) => {
+  const result = converter._deepCopy(dest_dict)
+  const keys = Object.keys(source_dict)
+  for (const name of keys) {
+    const merged_property = converter._merge_property(
+      merge_type,
+      name,
+      result[name],
+      source_dict[name],
+    )
+    if (merged_property !== undefined) {
+      result[name] = merged_property
+    }
+  }
+  return result
+}
+
+converter._scalar = (name, type, mode, description) => {
+  if (!name.match(BIGQUERY_FIELD_NAME_REGEXP)) {
+    throw new SchemaError(`Invalid field name: ${name}`)
+  }
+
+  const result = {
+    name,
+    type,
+    mode,
+  }
+
+  if (description) {
+    result.description = description
+  }
+
+  return result
+}
+
+converter._array = (name, node) => {
+  const items_with_description = converter._deepCopy(node.items)
+  if (_.has(items_with_description, 'description')) {
+    items_with_description.description = node.description
+  }
+  if (node.description) {
+    items_with_description.description = node.description
+  }
+  return converter._visit(name, items_with_description, 'REPEATED')
+}
+
+converter._allowsAdditionalProperties = (node) =>
+  node.additionalProperties !== false && node.unevaluatedProperties !== false
+
+converter._object = (name, node, mode) => {
+  let result = {
+    fields: [],
+  }
+  let fieldType = 'RECORD'
+  try {
+    if (
+      converter._allowsAdditionalProperties(node) &&
+      converter._options.preventAdditionalObjectProperties
+    ) {
+      throw new SchemaError(
+        'Objects must not have additional or unevaluated properties',
+        node,
+      )
+    }
+    if (
+      !_.isPlainObject(node.properties) ||
+      Object.keys(node.properties).length === 0
+    ) {
+      // Big Query can handle semi-structured data :
+      // https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-json#loading_semi-structured_json_data
+      fieldType = 'JSON'
+      return converter._scalar(name, fieldType, mode, node.description)
+    }
+
+    result = converter._scalar(name, fieldType, mode, node.description)
+
+    const required_properties = node.required || []
+    const properties = node.properties
+
+    let fields = Object.keys(properties).map((key) => {
+      const required = required_properties.includes(key)
+        ? 'REQUIRED'
+        : 'NULLABLE'
+      return converter._visit(key, properties[key], required)
+    })
+
+    // remove empty fields
+    fields = fields.filter(function (field) {
+      return field != null
+    })
+    result.fields = fields
+  } catch (e) {
+    if (!converter._options.continueOnError) {
+      throw e
+    }
+    logger.warn(e)
+  }
+
+  return result
+}
+
+converter._bigQueryType = (node, type) => {
+  // handle string formats
+  let actualType = type
+  const format = node.format
+  if (type === 'string' && ['date-time', 'date', 'time'].includes(format)) {
+    actualType = format
+  }
+  const bqType = JSON_SCHEMA_TO_BIGQUERY_TYPE_DICT[actualType]
+  if (!bqType) {
+    throw new SchemaError(`Invalid type given: ${type}`, node)
+  }
+  return bqType
+}
+
+converter._simple = (name, type, node, mode) => {
+  if (type === 'array') {
+    return converter._array(name, node)
+  }
+  if (type === 'object') {
+    return converter._object(name, node, mode)
+  }
+
+  const bqType = converter._bigQueryType(node, type)
+  return converter._scalar(name, bqType, mode, node.description)
+}
+
+converter._visit = (name, node, mode = 'NULLABLE') => {
+  let merged_node = node
+  for (const x_of of OFS) {
+    if (_.has(node, x_of)) {
+      merged_node = converter._merge_dicts_array(x_of, node, _.get(node, x_of))
+      delete merged_node[x_of]
+    }
+  }
+  let type_ = merged_node.type
+  let actual_mode = mode
+  if (Array.isArray(type_)) {
+    const non_null_types = type_.filter((scalar_type) => scalar_type !== 'null')
+    if (non_null_types.length > 1) {
+      throw new SchemaError('Union type not supported', node)
+    }
+    // When mode is REPEATED, we want to leave it, even if it is NULLABLE
+    if (type_.includes('null') && actual_mode !== 'REPEATED') {
+      actual_mode = 'NULLABLE'
+    }
+    type_ = non_null_types[0]
+  }
+  return converter._simple(name, type_, merged_node, actual_mode)
+}
+
+converter.run = (input_schema, options = {}) => {
+  converter._options = options
+  return {
+    schema: {
+      fields: converter._visit('root', input_schema).fields,
+    },
+  }
+}

package/dist/ai-context/internal/examples/v2/google-bigquery/app/jsonschema-bigquery-custom/src/errors.js

@@ -0,0 +1,21 @@
+const utils = require('./utils')
+
+class SchemaError extends Error {
+  constructor(message, node) {
+    super(message)
+
+    // Maintains proper stack trace for where our error was thrown (only available on V8)
+    if (Error.captureStackTrace) {
+      Error.captureStackTrace(this, SchemaError)
+    }
+
+    this.name = 'SchemaError'
+    if (node) {
+      this.message += ':\n' + utils.inspector(node)
+    }
+  }
+}
+
+module.exports = {
+  SchemaError,
+}