@naturalcycles/nodejs-lib 15.49.0 → 15.50.1

package/dist/validation/ajv/from-data/generateJsonSchemaFromData.d.ts

@@ -0,0 +1,8 @@
+import { type AnyObject } from '@naturalcycles/js-lib/types';
+import type { JsonSchema } from '../jsonSchemaBuilder.js';
+/**
+ * Each row must be an object (current limitation).
+ *
+ * `additionalProperties` is set to `true`, cause it's safer.
+ */
+export declare function generateJsonSchemaFromData<T extends AnyObject = AnyObject>(rows: AnyObject[]): JsonSchema<T>;

package/dist/validation/ajv/from-data/generateJsonSchemaFromData.js

@@ -0,0 +1,87 @@
+import { _uniq } from '@naturalcycles/js-lib/array';
+import { _stringMapEntries } from '@naturalcycles/js-lib/types';
+/**
+ * Each row must be an object (current limitation).
+ *
+ * `additionalProperties` is set to `true`, cause it's safer.
+ */
+export function generateJsonSchemaFromData(rows) {
+    return objectToJsonSchema(rows);
+}
+function objectToJsonSchema(rows) {
+    const typesByKey = {};
+    rows.forEach(r => {
+        Object.keys(r).forEach(key => {
+            typesByKey[key] ||= new Set();
+            typesByKey[key].add(getTypeOfValue(r[key]));
+        });
+    });
+    const s = {
+        type: 'object',
+        properties: {},
+        required: [],
+        additionalProperties: true,
+    };
+    _stringMapEntries(typesByKey).forEach(([key, types]) => {
+        const schema = mergeTypes([...types], rows.map(r => r[key]));
+        if (!schema)
+            return;
+        s.properties[key] = schema;
+    });
+    // console.log(typesByKey)
+    return s;
+}
+function mergeTypes(types, samples) {
+    // skip "undefined" types
+    types = types.filter(t => t !== 'undefined');
+    if (!types.length)
+        return undefined;
+    if (types.length > 1) {
+        // oneOf
+        const s = {
+            oneOf: types.map(type => mergeTypes([type], samples)),
+        };
+        return s;
+    }
+    const type = types[0];
+    if (type === 'null') {
+        return {
+            type: 'null',
+        };
+    }
+    if (type === 'boolean') {
+        return {
+            type: 'boolean',
+        };
+    }
+    if (type === 'string') {
+        return {
+            type: 'string',
+        };
+    }
+    if (type === 'number') {
+        return {
+            type: 'number',
+        };
+    }
+    if (type === 'object') {
+        return objectToJsonSchema(samples.filter((r) => r && typeof r === 'object'));
+    }
+    if (type === 'array') {
+        // possible feature: detect if it's a tuple
+        // currently assume no-tuple
+        const items = samples.filter(r => Array.isArray(r)).flat();
+        const itemTypes = _uniq(items.map(i => getTypeOfValue(i)));
+        return {
+            type: 'array',
+            items: mergeTypes(itemTypes, items),
+        };
+    }
+}
+function getTypeOfValue(v) {
+    if (v === null)
+        return 'null';
+    if (Array.isArray(v))
+        return 'array';
+    return typeof v;
+}

package/dist/validation/ajv/getAjv.js

@@ -220,23 +220,42 @@ export function createAjv(opt) {
     ajv.addKeyword({
         keyword: 'email',
         type: 'string',
-        modifying: false,
+        modifying: true,
         errors: true,
         schemaType: 'object',
         validate: function validate(opt, data, _schema, ctx) {
             const { checkTLD } = opt;
-            if (!checkTLD)
-                return true;
-            const tld = _substringAfterLast(data, '.');
-            if (validTLDs.has(tld))
-                return true;
-            validate.errors = [
-                {
-                    instancePath: ctx?.instancePath ?? '',
-                    message: `has an invalid TLD`,
-                },
-            ];
-            return false;
+            const cleanData = data.trim();
+            // from `ajv-formats`
+            const EMAIL_REGEX = /^[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/i;
+            const result = cleanData.match(EMAIL_REGEX);
+            if (!result) {
+                ;
+                validate.errors = [
+                    {
+                        instancePath: ctx?.instancePath ?? '',
+                        message: `is not a valid email address`,
+                    },
+                ];
+                return false;
+            }
+            if (checkTLD) {
+                const tld = _substringAfterLast(cleanData, '.');
+                if (!validTLDs.has(tld)) {
+                    ;
+                    validate.errors = [
+                        {
+                            instancePath: ctx?.instancePath ?? '',
+                            message: `has an invalid TLD`,
+                        },
+                    ];
+                    return false;
+                }
+            }
+            if (ctx?.parentData && ctx.parentDataProperty) {
+                ctx.parentData[ctx.parentDataProperty] = cleanData;
+            }
+            return true;
         },
     });
     ajv.addKeyword({

package/dist/validation/ajv/index.d.ts

@@ -1,6 +1,7 @@
 import Ajv from 'ajv';
 export * from './ajvSchema.js';
 export * from './ajvValidationError.js';
+export * from './from-data/generateJsonSchemaFromData.js';
 export * from './getAjv.js';
 export * from './jsonSchemaBuilder.js';
 export { Ajv };

package/dist/validation/ajv/index.js

@@ -1,6 +1,7 @@
 import Ajv from 'ajv';
 export * from './ajvSchema.js';
 export * from './ajvValidationError.js';
+export * from './from-data/generateJsonSchemaFromData.js';
 export * from './getAjv.js';
 export * from './jsonSchemaBuilder.js';
 export { Ajv };

package/dist/validation/ajv/jsonSchemaBuilder.js

@@ -217,9 +217,7 @@ export class JsonSchemaStringBuilder extends JsonSchemaAnyBuilder {
     email(opt) {
         const defaultOptions = { checkTLD: true };
         _objectAssign(this.schema, { email: { ...defaultOptions, ...opt } });
-        // from `ajv-formats`
-        const regex = /^[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/i;
-        return this.regex(regex, { msg: 'is not a valid email address' }).trim().toLowerCase();
+        return this.trim().toLowerCase();
     }
     trim() {
         _objectAssign(this.schema, { transform: { ...this.schema.transform, trim: true } });
@@ -496,7 +494,7 @@ export class JsonSchemaObjectBuilder extends JsonSchemaAnyBuilder {
     /**
      * Extends the current schema with `id`, `created` and `updated` according to NC DB conventions.
      */
-    // eslint-disable-next-line @typescript-eslint/explicit-function-return-type, @typescript-eslint/explicit-module-boundary-types
+    // oxlint-disable-next-line @typescript-eslint/explicit-function-return-type, @typescript-eslint/explicit-module-boundary-types
     dbEntity() {
         return this.extend({
             id: j.string(),
@@ -558,7 +556,7 @@ export class JsonSchemaObjectInferringBuilder extends JsonSchemaAnyBuilder {
     /**
      * Extends the current schema with `id`, `created` and `updated` according to NC DB conventions.
      */
-    // eslint-disable-next-line @typescript-eslint/explicit-function-return-type, @typescript-eslint/explicit-module-boundary-types
+    // oxlint-disable-next-line @typescript-eslint/explicit-function-return-type, @typescript-eslint/explicit-module-boundary-types
     dbEntity() {
         return this.extend({
             id: j.string(),

package/dist/yargs/yargs.util.js

@@ -5,7 +5,7 @@ import { hideBin } from 'yargs/helpers';
  * Quick yargs helper to make it work in esm.
  * It also allows to not have yargs and `@types/yargs` to be declared as dependencies.
  */
-// eslint-disable-next-line @typescript-eslint/explicit-function-return-type, @typescript-eslint/explicit-module-boundary-types
+// oxlint-disable-next-line @typescript-eslint/explicit-function-return-type, @typescript-eslint/explicit-module-boundary-types
 export function _yargs() {
     return yargs(hideBin(process.argv));
 }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@naturalcycles/nodejs-lib",
   "type": "module",
-  "version": "15.49.0",
+  "version": "15.50.1",
   "dependencies": {
     "@naturalcycles/js-lib": "^15",
     "@types/js-yaml": "^4",

package/src/validation/ajv/from-data/generateJsonSchemaFromData.ts

@@ -0,0 +1,112 @@
+import { _uniq } from '@naturalcycles/js-lib/array'
+import { _stringMapEntries, type AnyObject, type StringMap } from '@naturalcycles/js-lib/types'
+import type { JsonSchema } from '../jsonSchemaBuilder.js'
+
+type PrimitiveType = 'undefined' | 'null' | 'boolean' | 'string' | 'number'
+type Type = PrimitiveType | 'array' | 'object'
+
+/**
+ * Each row must be an object (current limitation).
+ *
+ * `additionalProperties` is set to `true`, cause it's safer.
+ */
+export function generateJsonSchemaFromData<T extends AnyObject = AnyObject>(
+  rows: AnyObject[],
+): JsonSchema<T> {
+  return objectToJsonSchema<T>(rows as any)
+}
+
+function objectToJsonSchema<T extends AnyObject>(rows: AnyObject[]): JsonSchema<T> {
+  const typesByKey: StringMap<Set<Type>> = {}
+
+  rows.forEach(r => {
+    Object.keys(r).forEach(key => {
+      typesByKey[key] ||= new Set<Type>()
+      typesByKey[key].add(getTypeOfValue(r[key]))
+    })
+  })
+
+  const s: JsonSchema<T> = {
+    type: 'object',
+    properties: {} as any,
+    required: [],
+    additionalProperties: true,
+  }
+
+  _stringMapEntries(typesByKey).forEach(([key, types]) => {
+    const schema = mergeTypes(
+      [...types],
+      rows.map(r => r[key]),
+    )
+    if (!schema) return
+    s.properties![key as keyof T] = schema as any
+  })
+
+  // console.log(typesByKey)
+
+  return s
+}
+
+function mergeTypes(types: Type[], samples: any[]): JsonSchema | undefined {
+  // skip "undefined" types
+  types = types.filter(t => t !== 'undefined')
+
+  if (!types.length) return undefined
+
+  if (types.length > 1) {
+    // oneOf
+    const s: JsonSchema = {
+      oneOf: types.map(type => mergeTypes([type], samples)!),
+    }
+
+    return s
+  }
+
+  const type = types[0]!
+
+  if (type === 'null') {
+    return {
+      type: 'null',
+    } as JsonSchema
+  }
+
+  if (type === 'boolean') {
+    return {
+      type: 'boolean',
+    } as JsonSchema
+  }
+
+  if (type === 'string') {
+    return {
+      type: 'string',
+    } as JsonSchema
+  }
+
+  if (type === 'number') {
+    return {
+      type: 'number',
+    } as JsonSchema
+  }
+
+  if (type === 'object') {
+    return objectToJsonSchema(samples.filter((r: any) => r && typeof r === 'object'))
+  }
+
+  if (type === 'array') {
+    // possible feature: detect if it's a tuple
+    // currently assume no-tuple
+    const items = samples.filter(r => Array.isArray(r)).flat()
+    const itemTypes = _uniq(items.map(i => getTypeOfValue(i)))
+
+    return {
+      type: 'array',
+      items: mergeTypes(itemTypes, items),
+    } as JsonSchema
+  }
+}
+
+function getTypeOfValue(v: any): Type {
+  if (v === null) return 'null'
+  if (Array.isArray(v)) return 'array'
+  return typeof v as Type
+}

package/src/validation/ajv/getAjv.ts

@@ -244,22 +244,46 @@ export function createAjv(opt?: Options): Ajv {
   ajv.addKeyword({
     keyword: 'email',
     type: 'string',
-    modifying: false,
+    modifying: true,
     errors: true,
     schemaType: 'object',
     validate: function validate(opt: JsonSchemaStringEmailOptions, data: string, _schema, ctx) {
       const { checkTLD } = opt
-      if (!checkTLD) return true
+      const cleanData = data.trim()
 
-      const tld = _substringAfterLast(data, '.')
-      if (validTLDs.has(tld)) return true
-      ;(validate as any).errors = [
-        {
-          instancePath: ctx?.instancePath ?? '',
-          message: `has an invalid TLD`,
-        },
-      ]
-      return false
+      // from `ajv-formats`
+      const EMAIL_REGEX =
+        /^[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/i
+      const result = cleanData.match(EMAIL_REGEX)
+
+      if (!result) {
+        ;(validate as any).errors = [
+          {
+            instancePath: ctx?.instancePath ?? '',
+            message: `is not a valid email address`,
+          },
+        ]
+        return false
+      }
+
+      if (checkTLD) {
+        const tld = _substringAfterLast(cleanData, '.')
+        if (!validTLDs.has(tld)) {
+          ;(validate as any).errors = [
+            {
+              instancePath: ctx?.instancePath ?? '',
+              message: `has an invalid TLD`,
+            },
+          ]
+          return false
+        }
+      }
+
+      if (ctx?.parentData && ctx.parentDataProperty) {
+        ctx.parentData[ctx.parentDataProperty] = cleanData
+      }
+
+      return true
     },
   })
 

package/src/validation/ajv/index.ts

@@ -2,6 +2,7 @@ import Ajv from 'ajv'
 
 export * from './ajvSchema.js'
 export * from './ajvValidationError.js'
+export * from './from-data/generateJsonSchemaFromData.js'
 export * from './getAjv.js'
 export * from './jsonSchemaBuilder.js'
 

package/src/validation/ajv/jsonSchemaBuilder.ts

@@ -302,11 +302,7 @@ export class JsonSchemaStringBuilder<
   email(opt?: Partial<JsonSchemaStringEmailOptions>): this {
     const defaultOptions: JsonSchemaStringEmailOptions = { checkTLD: true }
     _objectAssign(this.schema, { email: { ...defaultOptions, ...opt } })
-
-    // from `ajv-formats`
-    const regex =
-      /^[a-z0-9!#$%&'*+/=?^_`{|}~-]+(?:\.[a-z0-9!#$%&'*+/=?^_`{|}~-]+)*@(?:[a-z0-9](?:[a-z0-9-]*[a-z0-9])?\.)+[a-z0-9](?:[a-z0-9-]*[a-z0-9])?$/i
-    return this.regex(regex, { msg: 'is not a valid email address' }).trim().toLowerCase()
+    return this.trim().toLowerCase()
   }
 
   trim(): this {
@@ -677,7 +673,7 @@ export class JsonSchemaObjectBuilder<
   /**
    * Extends the current schema with `id`, `created` and `updated` according to NC DB conventions.
    */
-  // eslint-disable-next-line @typescript-eslint/explicit-function-return-type, @typescript-eslint/explicit-module-boundary-types
+  // oxlint-disable-next-line @typescript-eslint/explicit-function-return-type, @typescript-eslint/explicit-module-boundary-types
   dbEntity() {
     return this.extend({
       id: j.string(),
@@ -805,7 +801,7 @@ export class JsonSchemaObjectInferringBuilder<
   /**
    * Extends the current schema with `id`, `created` and `updated` according to NC DB conventions.
    */
-  // eslint-disable-next-line @typescript-eslint/explicit-function-return-type, @typescript-eslint/explicit-module-boundary-types
+  // oxlint-disable-next-line @typescript-eslint/explicit-function-return-type, @typescript-eslint/explicit-module-boundary-types
   dbEntity() {
     return this.extend({
       id: j.string(),

package/src/yargs/yargs.util.ts

@@ -6,7 +6,7 @@ import { hideBin } from 'yargs/helpers'
  * Quick yargs helper to make it work in esm.
  * It also allows to not have yargs and `@types/yargs` to be declared as dependencies.
  */
-// eslint-disable-next-line @typescript-eslint/explicit-function-return-type, @typescript-eslint/explicit-module-boundary-types
+// oxlint-disable-next-line @typescript-eslint/explicit-function-return-type, @typescript-eslint/explicit-module-boundary-types
 export function _yargs() {
   return yargs(hideBin(process.argv))
 }