@mongodb-js/mongodb-schema 12.7.1

package/bin/mongodb-schema

@@ -0,0 +1,231 @@
+#!/usr/bin/env node
+
+/* eslint-disable no-console */
+/* eslint-disable @typescript-eslint/no-var-requires */
+
+const { SchemaAnalyzer, schemaStats: _schemaStats } = require('../');
+const schemaStats = _schemaStats.default;
+
+const { MongoClient } = require('mongodb');
+const toNS = require('mongodb-ns');
+const yaml = require('js-yaml');
+const pkg = require('../package.json');
+const Table = require('cli-table');
+const numeral = require('numeral');
+const { EJSON } = require('bson');
+const ProgressBar = require('progress');
+
+const yargs = require('yargs/yargs');
+const { hideBin } = require('yargs/helpers');
+
+const argv = yargs(hideBin(process.argv))
+  .strict()
+  .usage('Usage: $0 <uri> <ns> [--format=<json|yaml|table> --number=<n>]')
+  .demand(2)
+  .option('n', {
+    alias: 'number',
+    default: 100,
+    describe: 'The number of documents to return.',
+  })
+  .option('f', {
+    alias: 'format',
+    default: 'json',
+    describe: 'The output format.',
+    choices: ['json', 'yaml', 'table'],
+  })
+  .option('o', {
+    alias: 'output',
+    type: 'boolean',
+    describe: 'Print the computed schema to stdout.',
+    default: true,
+  })
+  .option('s', {
+    alias: 'stats',
+    type: 'boolean',
+    describe: 'print schema statistics to stderr',
+  })
+  .option('p', {
+    alias: 'promote',
+    type: 'boolean',
+    default: true,
+    describe: 'promote values to Javascript numbers.',
+  })
+  .options('t', {
+    alias: 'semantic-types',
+    type: 'boolean',
+    default: false,
+    describe:
+      'semantic type detection, currently supported are emails and geojson',
+  })
+  .option('values', {
+    type: 'boolean',
+    default: true,
+    describe: 'enables the collection of sample values',
+  })
+  .option('sampling', {
+    type: 'boolean',
+    default: true,
+    describe: 'use random sampling on the collection.',
+  })
+  .describe('version', 'Show version.')
+  .alias('h', 'help')
+  .describe('h', 'Show this screen.')
+  .help('h')
+  .wrap(100)
+  .example(
+    '$0 localhost:27017 mongodb.fanclub --number 1000 --stats ' + '--no-output',
+    'analyze 1000 docs from the mongodb.fanclub ' +
+      'collection and only show statistics.',
+  )
+  .example(
+    '$0 localhost:27017 test.foo --format table',
+    'analyze 100 docs from the test.foo collection and print ' +
+      'the schema in table form.',
+  ).argv;
+
+let uri = argv._[0];
+if (!uri.startsWith('mongodb://') && !uri.startsWith('mongodb+srv://')) {
+  uri = 'mongodb://' + uri;
+}
+const sampleSize = parseInt(argv.number, 10);
+
+if (argv.version) {
+  console.error(pkg.version);
+  process.exit(1);
+}
+
+function addTableRow(table, field) {
+  table.push([
+    field.path,
+    field.type,
+    numeral(field.probability).format('0.000%'),
+  ]);
+
+  if (field.fields) {
+    field.fields.map(function (child) {
+      addTableRow(table, child);
+    });
+  }
+
+  if (field.arrayFields) {
+    field.arrayFields.map(function (child) {
+      addTableRow(table, child);
+    });
+  }
+}
+
+function getTable(schema) {
+  const table = new Table({
+    head: ['Path', 'Type', 'Probability'],
+    colWidths: [50, 30, 20],
+  });
+  schema.fields.map(function (field) {
+    addTableRow(table, field);
+  });
+  return table;
+}
+
+const bar = new ProgressBar('analyzing [:bar] :percent :etas ', {
+  total: argv.number,
+  width: 60,
+  complete: '=',
+  incomplete: ' ',
+  clear: true,
+});
+
+function sample(collection, size = 1000) {
+  return collection.aggregate(
+    [
+      {
+        $sample: {
+          size,
+        },
+      },
+    ],
+    {
+      allowDiskUse: true,
+    },
+  );
+}
+
+const client = new MongoClient(uri);
+
+(async function main() {
+  try {
+    await client.connect();
+  } catch (err) {
+    if (err) {
+      console.error('Failed to connect to MongoDB: ', err);
+      process.exit(1);
+    }
+  }
+
+  const ns = toNS(argv._[1]);
+  const db = client.db(ns.database);
+  let ts;
+
+  const options = {
+    size: sampleSize,
+    query: {},
+    promoteValues: argv.promote,
+  };
+
+  const schemaOptions = {
+    storeValues: argv.values,
+    semanticTypes: argv.semanticTypes,
+  };
+
+  const analyzer = new SchemaAnalyzer(schemaOptions);
+  try {
+    const input = argv.sampling
+      ? sample(db.collection(ns.collection), sampleSize)
+      : db
+          .collection(ns.collection)
+          .find(options.query, {
+            promoteValues: options.promoteValues,
+          })
+          .limit(options.size);
+
+    for await (const doc of input) {
+      bar.tick();
+      analyzer.analyzeDoc(doc);
+    }
+  } catch (err) {
+    console.error('error:', err.message);
+    process.exit(1);
+  }
+
+  const schema = analyzer.getResult();
+
+  if (argv.output) {
+    let output = '';
+    if (argv.format === 'yaml') {
+      output = yaml.dump(schema);
+    } else if (argv.format === 'table') {
+      output = getTable(schema).toString();
+    } else {
+      output = EJSON.stringify(schema, null, 2);
+    }
+    console.log(output);
+  }
+
+  if (argv.stats) {
+    let branchOutput = '[';
+    const branchingFactors = schemaStats.branch(schema);
+    if (branchingFactors.length > 20) {
+      branchOutput += `${branchingFactors.slice(0, 20).join(',')},...] (top 20 shown)`;
+    } else {
+      branchOutput += branchingFactors.join(',') + ']';
+    }
+
+    console.error('toplevel fields:', schema.fields.length);
+    console.error('branching factors:', branchOutput);
+    console.error('schema width: ' + schemaStats.width(schema));
+    console.error('schema depth: ' + schemaStats.depth(schema));
+  }
+
+  console.dir(analyzer.getSchemaPaths());
+  console.dir(analyzer.getSimplifiedSchema(), { depth: null });
+
+  client.close();
+})();

package/dist/index.d.ts

@@ -0,0 +1,14 @@
+import type { SchemaAccessor } from './schema-accessor';
+import { SchemaAnalyzer } from './schema-analyzer';
+import type { ArraySchemaType, BaseSchemaType, ConstantSchemaType, DocumentSchemaType, PrimitiveSchemaType, SchemaType, Schema as InternalSchema, SchemaField, SchemaParseOptions, SimplifiedSchemaBaseType, SimplifiedSchemaArrayType, SimplifiedSchemaDocumentType, SimplifiedSchemaType, SimplifiedSchemaField, SimplifiedSchema } from './schema-analyzer';
+import * as schemaStats from './stats';
+import type { AnyIterable, StandardJSONSchema, MongoDBJSONSchema, ExpandedJSONSchema, JSONSchema } from './types';
+import { toTypescriptTypeDefinition } from './to-typescript';
+declare function analyzeDocuments(source: AnyIterable, options?: SchemaParseOptions): Promise<SchemaAccessor>;
+declare function parseSchema(source: AnyIterable, options?: Partial<SchemaParseOptions>): Promise<InternalSchema>;
+declare function getSchemaPaths(source: AnyIterable): Promise<string[][]>;
+declare function getSimplifiedSchema(source: AnyIterable): Promise<SimplifiedSchema>;
+export default parseSchema;
+export type { ArraySchemaType, BaseSchemaType, ConstantSchemaType, DocumentSchemaType, PrimitiveSchemaType, SchemaType, InternalSchema as Schema, InternalSchema, SchemaField, SchemaParseOptions, SimplifiedSchemaBaseType, SimplifiedSchemaArrayType, SimplifiedSchemaDocumentType, SimplifiedSchemaType, SimplifiedSchemaField, SimplifiedSchema, StandardJSONSchema, MongoDBJSONSchema, ExpandedJSONSchema, JSONSchema, SchemaAccessor, };
+export { parseSchema, analyzeDocuments, getSchemaPaths, getSimplifiedSchema, SchemaAnalyzer, schemaStats, toTypescriptTypeDefinition, };
+//# sourceMappingURL=index.d.ts.map

package/dist/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAExD,OAAO,EAA8B,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAC/E,OAAO,KAAK,EACV,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,kBAAkB,EAClB,mBAAmB,EACnB,UAAU,EACV,MAAM,IAAI,cAAc,EACxB,WAAW,EACX,kBAAkB,EAClB,wBAAwB,EACxB,yBAAyB,EACzB,4BAA4B,EAC5B,oBAAoB,EACpB,qBAAqB,EACrB,gBAAgB,EACjB,MAAM,mBAAmB,CAAC;AAI3B,OAAO,KAAK,WAAW,MAAM,SAAS,CAAC;AACvC,OAAO,KAAK,EACV,WAAW,EACX,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,UAAU,EACX,MAAM,SAAS,CAAC;AACjB,OAAO,EAAE,0BAA0B,EAAE,MAAM,iBAAiB,CAAC;AAK7D,iBAAe,gBAAgB,CAC7B,MAAM,EAAE,WAAW,EACnB,OAAO,CAAC,EAAE,kBAAkB,GAC3B,OAAO,CAAC,cAAc,CAAC,CASzB;AAMD,iBAAe,WAAW,CACxB,MAAM,EAAE,WAAW,EACnB,OAAO,CAAC,EAAE,OAAO,CAAC,kBAAkB,CAAC,GACpC,OAAO,CAAC,cAAc,CAAC,CAEzB;AAGD,iBAAe,cAAc,CAAC,MAAM,EAAE,WAAW,GAAG,OAAO,CAAC,MAAM,EAAE,EAAE,CAAC,CAEtE;AAGD,iBAAe,mBAAmB,CAChC,MAAM,EAAE,WAAW,GAClB,OAAO,CAAC,gBAAgB,CAAC,CAE3B;AAED,eAAe,WAAW,CAAC;AAE3B,YAAY,EACV,eAAe,EACf,cAAc,EACd,kBAAkB,EAClB,kBAAkB,EAClB,mBAAmB,EACnB,UAAU,EACV,cAAc,IAAI,MAAM,EACxB,cAAc,EACd,WAAW,EACX,kBAAkB,EAClB,wBAAwB,EACxB,yBAAyB,EACzB,4BAA4B,EAC5B,oBAAoB,EACpB,qBAAqB,EACrB,gBAAgB,EAChB,kBAAkB,EAClB,iBAAiB,EACjB,kBAAkB,EAClB,UAAU,EACV,cAAc,GACf,CAAC;AAEF,OAAO,EACL,WAAW,EACX,gBAAgB,EAChB,cAAc,EACd,mBAAmB,EACnB,cAAc,EACd,WAAW,EACX,0BAA0B,GAC3B,CAAC"}

package/dist/index.js

@@ -0,0 +1,69 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.toTypescriptTypeDefinition = exports.schemaStats = exports.SchemaAnalyzer = void 0;
+exports.parseSchema = parseSchema;
+exports.analyzeDocuments = analyzeDocuments;
+exports.getSchemaPaths = getSchemaPaths;
+exports.getSimplifiedSchema = getSimplifiedSchema;
+const schema_accessor_1 = require("./schema-accessor");
+const schema_analyzer_1 = require("./schema-analyzer");
+Object.defineProperty(exports, "SchemaAnalyzer", { enumerable: true, get: function () { return schema_analyzer_1.SchemaAnalyzer; } });
+const internal_to_expanded_1 = require("./schema-converters/internal-to-expanded");
+const internal_to_mongodb_1 = require("./schema-converters/internal-to-mongodb");
+const internal_to_standard_1 = require("./schema-converters/internal-to-standard");
+const schemaStats = __importStar(require("./stats"));
+exports.schemaStats = schemaStats;
+const to_typescript_1 = require("./to-typescript");
+Object.defineProperty(exports, "toTypescriptTypeDefinition", { enumerable: true, get: function () { return to_typescript_1.toTypescriptTypeDefinition; } });
+async function analyzeDocuments(source, options) {
+    const internalSchema = (await (0, schema_analyzer_1.getCompletedSchemaAnalyzer)(source, options)).getResult();
+    return new schema_accessor_1.InternalSchemaBasedAccessor(internalSchema, {
+        internalToStandard: internal_to_standard_1.convertInternalToStandard,
+        internalToMongoDB: internal_to_mongodb_1.convertInternalToMongodb,
+        internalToExpanded: internal_to_expanded_1.convertInternalToExpanded,
+    });
+}
+async function parseSchema(source, options) {
+    return (await (0, schema_analyzer_1.getCompletedSchemaAnalyzer)(source, options)).getResult();
+}
+async function getSchemaPaths(source) {
+    return (await (0, schema_analyzer_1.getCompletedSchemaAnalyzer)(source)).getSchemaPaths();
+}
+async function getSimplifiedSchema(source) {
+    return (await (0, schema_analyzer_1.getCompletedSchemaAnalyzer)(source)).getSimplifiedSchema();
+}
+exports.default = parseSchema;
+//# sourceMappingURL=index.js.map

package/dist/index.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.js","sourceRoot":"","sources":["../src/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAoGE,kCAAW;AACX,4CAAgB;AAChB,wCAAc;AACd,kDAAmB;AAtGrB,uDAAgE;AAChE,uDAA+E;AAsG7E,+FAtGmC,gCAAc,OAsGnC;AApFhB,mFAAqF;AACrF,iFAAmF;AACnF,mFAAqF;AACrF,qDAAuC;AAkFrC,kCAAW;AA1Eb,mDAA6D;AA2E3D,2GA3EO,0CAA0B,OA2EP;AAtE5B,KAAK,UAAU,gBAAgB,CAC7B,MAAmB,EACnB,OAA4B;IAE5B,MAAM,cAAc,GAAG,CACrB,MAAM,IAAA,4CAA0B,EAAC,MAAM,EAAE,OAAO,CAAC,CAClD,CAAC,SAAS,EAAE,CAAC;IACd,OAAO,IAAI,6CAA2B,CAAC,cAAc,EAAE;QACrD,kBAAkB,EAAE,gDAAyB;QAC7C,iBAAiB,EAAE,8CAAwB;QAC3C,kBAAkB,EAAE,gDAAyB;KAC9C,CAAC,CAAC;AACL,CAAC;AAMD,KAAK,UAAU,WAAW,CACxB,MAAmB,EACnB,OAAqC;IAErC,OAAO,CAAC,MAAM,IAAA,4CAA0B,EAAC,MAAM,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC;AACzE,CAAC;AAGD,KAAK,UAAU,cAAc,CAAC,MAAmB;IAC/C,OAAO,CAAC,MAAM,IAAA,4CAA0B,EAAC,MAAM,CAAC,CAAC,CAAC,cAAc,EAAE,CAAC;AACrE,CAAC;AAGD,KAAK,UAAU,mBAAmB,CAChC,MAAmB;IAEnB,OAAO,CAAC,MAAM,IAAA,4CAA0B,EAAC,MAAM,CAAC,CAAC,CAAC,mBAAmB,EAAE,CAAC;AAC1E,CAAC;AAED,kBAAe,WAAW,CAAC"}

package/dist/schema-accessor.d.ts

@@ -0,0 +1,30 @@
+import type { Schema as InternalSchema } from './schema-analyzer';
+import type { ExpandedJSONSchema, MongoDBJSONSchema, SchemaConverterFn, StandardJSONSchema } from './types';
+type Options = {
+    signal?: AbortSignal;
+};
+export interface SchemaAccessor {
+    getStandardJsonSchema: (options?: Options) => Promise<StandardJSONSchema>;
+    getMongoDBJsonSchema: (options?: Options) => Promise<MongoDBJSONSchema>;
+    getExpandedJSONSchema: (options?: Options) => Promise<ExpandedJSONSchema>;
+    getInternalSchema: (options?: Options) => Promise<InternalSchema>;
+}
+export type InternalConverters = {
+    internalToStandard: SchemaConverterFn<InternalSchema, StandardJSONSchema>;
+    internalToExpanded: SchemaConverterFn<InternalSchema, ExpandedJSONSchema>;
+    internalToMongoDB: SchemaConverterFn<InternalSchema, MongoDBJSONSchema>;
+};
+export declare class InternalSchemaBasedAccessor implements SchemaAccessor {
+    private internalSchema;
+    private standardJSONSchema?;
+    private mongodbJSONSchema?;
+    private expandedJSONSchema?;
+    private converters;
+    constructor(internalSchema: InternalSchema, converters: InternalConverters);
+    getInternalSchema(): Promise<InternalSchema>;
+    getStandardJsonSchema(options?: Options): Promise<StandardJSONSchema>;
+    getMongoDBJsonSchema(options?: Options): Promise<MongoDBJSONSchema>;
+    getExpandedJSONSchema(options?: Options): Promise<ExpandedJSONSchema>;
+}
+export {};
+//# sourceMappingURL=schema-accessor.d.ts.map

package/dist/schema-accessor.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-accessor.d.ts","sourceRoot":"","sources":["../src/schema-accessor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,IAAI,cAAc,EAAE,MAAM,mBAAmB,CAAC;AAClE,OAAO,KAAK,EACV,kBAAkB,EAClB,iBAAiB,EACjB,iBAAiB,EACjB,kBAAkB,EACnB,MAAM,SAAS,CAAC;AAEjB,KAAK,OAAO,GAAG;IACb,MAAM,CAAC,EAAE,WAAW,CAAC;CACtB,CAAC;AAEF,MAAM,WAAW,cAAc;IAC7B,qBAAqB,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC1E,oBAAoB,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,OAAO,CAAC,iBAAiB,CAAC,CAAC;IACxE,qBAAqB,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,OAAO,CAAC,kBAAkB,CAAC,CAAC;IAC1E,iBAAiB,EAAE,CAAC,OAAO,CAAC,EAAE,OAAO,KAAK,OAAO,CAAC,cAAc,CAAC,CAAC;CACnE;AASD,MAAM,MAAM,kBAAkB,GAAG;IAC/B,kBAAkB,EAAE,iBAAiB,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;IAC1E,kBAAkB,EAAE,iBAAiB,CAAC,cAAc,EAAE,kBAAkB,CAAC,CAAC;IAC1E,iBAAiB,EAAE,iBAAiB,CAAC,cAAc,EAAE,iBAAiB,CAAC,CAAC;CACzE,CAAC;AACF,qBAAa,2BAA4B,YAAW,cAAc;IAChE,OAAO,CAAC,cAAc,CAAiB;IACvC,OAAO,CAAC,kBAAkB,CAAC,CAAqB;IAChD,OAAO,CAAC,iBAAiB,CAAC,CAAoB;IAC9C,OAAO,CAAC,kBAAkB,CAAC,CAAqB;IAChD,OAAO,CAAC,UAAU,CAAqB;gBAE3B,cAAc,EAAE,cAAc,EAAE,UAAU,EAAE,kBAAkB;IAK1E,iBAAiB,IAAI,OAAO,CAAC,cAAc,CAAC;IAQtC,qBAAqB,CACzB,OAAO,GAAE,OAAY,GACpB,OAAO,CAAC,kBAAkB,CAAC;IAQxB,oBAAoB,CACxB,OAAO,GAAE,OAAY,GACpB,OAAO,CAAC,iBAAiB,CAAC;IAUvB,qBAAqB,CACzB,OAAO,GAAE,OAAY,GACpB,OAAO,CAAC,kBAAkB,CAAC;CAI/B"}

package/dist/schema-accessor.js

@@ -0,0 +1,30 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.InternalSchemaBasedAccessor = void 0;
+class InternalSchemaBasedAccessor {
+    internalSchema;
+    standardJSONSchema;
+    mongodbJSONSchema;
+    expandedJSONSchema;
+    converters;
+    constructor(internalSchema, converters) {
+        this.internalSchema = internalSchema;
+        this.converters = converters;
+    }
+    getInternalSchema() {
+        return Promise.resolve(this.internalSchema);
+    }
+    async getStandardJsonSchema(options = {}) {
+        return (this.standardJSONSchema ??=
+            await this.converters.internalToStandard(this.internalSchema, options));
+    }
+    async getMongoDBJsonSchema(options = {}) {
+        return (this.mongodbJSONSchema ??= await this.converters.internalToMongoDB(this.internalSchema, options));
+    }
+    async getExpandedJSONSchema(options = {}) {
+        return (this.expandedJSONSchema ??=
+            await this.converters.internalToExpanded(this.internalSchema, options));
+    }
+}
+exports.InternalSchemaBasedAccessor = InternalSchemaBasedAccessor;
+//# sourceMappingURL=schema-accessor.js.map

package/dist/schema-accessor.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-accessor.js","sourceRoot":"","sources":["../src/schema-accessor.ts"],"names":[],"mappings":";;;AA+BA,MAAa,2BAA2B;IAC9B,cAAc,CAAiB;IAC/B,kBAAkB,CAAsB;IACxC,iBAAiB,CAAqB;IACtC,kBAAkB,CAAsB;IACxC,UAAU,CAAqB;IAEvC,YAAY,cAA8B,EAAE,UAA8B;QACxE,IAAI,CAAC,cAAc,GAAG,cAAc,CAAC;QACrC,IAAI,CAAC,UAAU,GAAG,UAAU,CAAC;IAC/B,CAAC;IAED,iBAAiB;QACf,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,cAAc,CAAC,CAAC;IAC9C,CAAC;IAMD,KAAK,CAAC,qBAAqB,CACzB,UAAmB,EAAE;QAErB,OAAO,CAAC,IAAI,CAAC,kBAAkB;YAC7B,MAAM,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;IAC5E,CAAC;IAKD,KAAK,CAAC,oBAAoB,CACxB,UAAmB,EAAE;QAErB,OAAO,CAAC,IAAI,CAAC,iBAAiB,KAAK,MAAM,IAAI,CAAC,UAAU,CAAC,iBAAiB,CACxE,IAAI,CAAC,cAAc,EACnB,OAAO,CACR,CAAC,CAAC;IACL,CAAC;IAKD,KAAK,CAAC,qBAAqB,CACzB,UAAmB,EAAE;QAErB,OAAO,CAAC,IAAI,CAAC,kBAAkB;YAC7B,MAAM,IAAI,CAAC,UAAU,CAAC,kBAAkB,CAAC,IAAI,CAAC,cAAc,EAAE,OAAO,CAAC,CAAC,CAAC;IAC5E,CAAC;CACF;AAhDD,kEAgDC"}

package/dist/schema-analyzer.d.ts

@@ -0,0 +1,161 @@
+import Reservoir from 'reservoir';
+import type { Document, ObjectId, MinKey, MaxKey, Long, Double, Int32, Decimal128, BSONRegExp, BSONSymbol, Timestamp } from 'bson';
+import { Binary, Code } from 'bson';
+import type { AnyIterable } from './types';
+type TypeCastMap = {
+    Array: unknown[];
+    Binary: Binary;
+    Boolean: boolean;
+    Code: Code;
+    CodeWScope: Code;
+    Date: Date;
+    Decimal128: Decimal128;
+    Double: Double;
+    Int32: Int32;
+    Int64: Long;
+    MaxKey: MaxKey;
+    MinKey: MinKey;
+    Null: null;
+    Object: Record<string, unknown>;
+    ObjectId: ObjectId;
+    BSONRegExp: BSONRegExp;
+    String: string;
+    BSONSymbol: BSONSymbol;
+    Timestamp: Timestamp;
+    Undefined: undefined;
+};
+type TypeCastTypes = keyof TypeCastMap;
+type BSONValue = TypeCastMap[TypeCastTypes];
+export type BaseSchemaType = {
+    path: string[];
+    name: string;
+    count: number;
+    probability: number;
+    bsonType: string;
+    hasDuplicates?: boolean;
+    unique?: number;
+};
+export type ConstantSchemaType = BaseSchemaType & {
+    name: 'Null' | 'Undefined';
+};
+export type PrimitiveSchemaType = BaseSchemaType & {
+    name: 'String' | 'Number' | 'Int32' | 'Boolean' | 'Decimal128' | 'Long' | 'ObjectId' | 'Date' | 'RegExp' | 'Symbol' | 'MaxKey' | 'MinKey' | 'Binary' | 'Code' | 'Timestamp' | 'DBRef';
+    values: BSONValue[];
+};
+export type ArraySchemaType = BaseSchemaType & {
+    name: 'Array';
+    lengths: number[];
+    averageLength: number;
+    totalCount: number;
+    types: SchemaType[];
+};
+export type DocumentSchemaType = BaseSchemaType & {
+    name: 'Document';
+    fields: SchemaField[];
+};
+export type SchemaType = BaseSchemaType | ConstantSchemaType | PrimitiveSchemaType | ArraySchemaType | DocumentSchemaType;
+export type SchemaField = {
+    name: string;
+    count: number;
+    path: string[];
+    type: string | string[];
+    probability: number;
+    hasDuplicates: boolean;
+    types: SchemaType[];
+};
+export type Schema = {
+    count: number;
+    fields: SchemaField[];
+};
+type SchemaBSONType = Exclude<keyof TypeCastMap, 'Object'> | 'Document';
+type SchemaAnalysisBaseType = {
+    name: string;
+    path: string[];
+    bsonType: SchemaBSONType;
+    count: number;
+    values?: ReturnType<typeof Reservoir>;
+};
+type SchemaAnalysisNullType = SchemaAnalysisBaseType & {
+    name: 'Null';
+};
+type SchemaAnalysisPrimitiveType = SchemaAnalysisBaseType & {
+    name: 'String' | 'Number' | 'Int32' | 'Boolean' | 'Decimal128' | 'Long' | 'ObjectId' | 'Date' | 'RegExp' | 'Symbol' | 'MaxKey' | 'MinKey' | 'Binary' | 'Code' | 'Timestamp' | 'DBRef';
+};
+type SchemaAnalysisArrayType = SchemaAnalysisBaseType & {
+    name: 'Array';
+    lengths: number[];
+    types: SchemaAnalysisFieldTypes;
+};
+type SchemaAnalysisDocumentType = SchemaAnalysisBaseType & {
+    name: 'Document';
+    fields: SchemaAnalysisFieldsMap;
+};
+type SchemaAnalysisType = SchemaAnalysisBaseType | SchemaAnalysisNullType | SchemaAnalysisPrimitiveType | SchemaAnalysisArrayType | SchemaAnalysisDocumentType;
+type SchemaAnalysisFieldTypes = {
+    [fieldName: string]: SchemaAnalysisType;
+};
+type SchemaAnalysisField = {
+    name: string;
+    path: string[];
+    count: number;
+    types: SchemaAnalysisFieldTypes;
+};
+type SchemaAnalysisFieldsMap = {
+    [fieldName: string]: SchemaAnalysisField;
+};
+type SchemaAnalysisRoot = {
+    fields: SchemaAnalysisFieldsMap;
+    count: number;
+};
+type SemanticTypeFunction = (value: BSONValue, path?: string[]) => boolean;
+type SemanticTypeMap = {
+    [typeName: string]: SemanticTypeFunction | boolean;
+};
+type AllSchemaParseOptions = {
+    semanticTypes: boolean | SemanticTypeMap;
+    storeValues: boolean;
+    signal?: AbortSignal;
+    storedValuesLengthLimit: number;
+    distinctFieldsAbortThreshold?: number;
+};
+export type SchemaParseOptions = Partial<AllSchemaParseOptions>;
+export type SimplifiedSchemaBaseType = {
+    bsonType: SchemaBSONType;
+};
+export type SimplifiedSchemaArrayType = SimplifiedSchemaBaseType & {
+    bsonType: 'Array';
+    types: SimplifiedSchemaType[];
+};
+export type SimplifiedSchemaDocumentType = SimplifiedSchemaBaseType & {
+    bsonType: 'Document';
+    fields: SimplifiedSchema;
+};
+export type SimplifiedSchemaType = SimplifiedSchemaBaseType | SimplifiedSchemaArrayType | SimplifiedSchemaDocumentType;
+export type SimplifiedSchemaField = {
+    types: SimplifiedSchemaType[];
+};
+export type SimplifiedSchema = {
+    [fieldName: string]: SimplifiedSchemaField;
+};
+export declare class SchemaAnalyzer {
+    semanticTypes: SemanticTypeMap;
+    options: AllSchemaParseOptions;
+    documentsAnalyzed: number;
+    fieldsCount: number;
+    schemaAnalysisRoot: SchemaAnalysisRoot;
+    finalized: boolean;
+    schemaResult: Schema;
+    fieldAndTypeAnalysisCounter: number;
+    constructor(options?: SchemaParseOptions);
+    allowAbortDuringAnalysis(): Promise<void>;
+    increaseFieldCount(): void;
+    getSemanticType(value: BSONValue, path: string[]): string;
+    analyzeDoc(doc: Document): Promise<void>;
+    getResult(): Schema;
+    getSchemaPaths(): string[][];
+    getSimplifiedSchema(): SimplifiedSchema;
+}
+export declare function verifyStreamSource(source: AnyIterable): AnyIterable;
+export declare function getCompletedSchemaAnalyzer(source: AnyIterable, options?: SchemaParseOptions): Promise<SchemaAnalyzer>;
+export {};
+//# sourceMappingURL=schema-analyzer.d.ts.map

package/dist/schema-analyzer.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"schema-analyzer.d.ts","sourceRoot":"","sources":["../src/schema-analyzer.ts"],"names":[],"mappings":"AAAA,OAAO,SAAS,MAAM,WAAW,CAAC;AAClC,OAAO,KAAK,EACV,QAAQ,EACR,QAAQ,EACR,MAAM,EACN,MAAM,EACN,IAAI,EACJ,MAAM,EACN,KAAK,EACL,UAAU,EACV,UAAU,EACV,UAAU,EACV,SAAS,EACV,MAAM,MAAM,CAAC;AACd,OAAO,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,MAAM,CAAC;AAGpC,OAAO,KAAK,EAAE,WAAW,EAAE,MAAM,SAAS,CAAC;AAG3C,KAAK,WAAW,GAAG;IACjB,KAAK,EAAE,OAAO,EAAE,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;IACf,OAAO,EAAE,OAAO,CAAC;IACjB,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,IAAI,CAAC;IACjB,IAAI,EAAE,IAAI,CAAC;IACX,UAAU,EAAE,UAAU,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,KAAK,EAAE,KAAK,CAAC;IACb,KAAK,EAAE,IAAI,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;IACf,MAAM,EAAE,MAAM,CAAC;IACf,IAAI,EAAE,IAAI,CAAC;IACX,MAAM,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;IAChC,QAAQ,EAAE,QAAQ,CAAC;IACnB,UAAU,EAAE,UAAU,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,UAAU,CAAC;IACvB,SAAS,EAAE,SAAS,CAAC;IACrB,SAAS,EAAE,SAAS,CAAC;CACtB,CAAC;AAEF,KAAK,aAAa,GAAG,MAAM,WAAW,CAAC;AACvC,KAAK,SAAS,GAAG,WAAW,CAAC,aAAa,CAAC,CAAC;AAE5C,MAAM,MAAM,cAAc,GAAG;IAC3B,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,MAAM,CAAC;IAIjB,aAAa,CAAC,EAAE,OAAO,CAAC;IACxB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,cAAc,GAAG;IAChD,IAAI,EAAE,MAAM,GAAG,WAAW,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,mBAAmB,GAAG,cAAc,GAAG;IACjD,IAAI,EACA,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,SAAS,GACT,YAAY,GACZ,MAAM,GACN,UAAU,GACV,MAAM,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,WAAW,GACX,OAAO,CAAC;IACZ,MAAM,EAAE,SAAS,EAAE,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,eAAe,GAAG,cAAc,GAAG;IAC7C,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,aAAa,EAAE,MAAM,CAAC;IACtB,UAAU,EAAE,MAAM,CAAC;IAEnB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG,cAAc,GAAG;IAChD,IAAI,EAAE,UAAU,CAAC;IAEjB,MAAM,EAAE,WAAW,EAAE,CAAC;CACvB,CAAC;AAIF,MAAM,MAAM,UAAU,GAClB,cAAc,GACd,kBAAkB,GAClB,mBAAmB,GACnB,eAAe,GACf,kBAAkB,CAAC;AAEvB,MAAM,MAAM,WAAW,GAAG;IACxB,IAAI,EAAE,MAAM,CAAC;IACb,KAAK,EAAE,MAAM,CAAC;IACd,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,IAAI,EAAE,MAAM,GAAG,MAAM,EAAE,CAAC;IACxB,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,OAAO,CAAC;IACvB,KAAK,EAAE,UAAU,EAAE,CAAC;CACrB,CAAC;AAEF,MAAM,MAAM,MAAM,GAAG;IACnB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,EAAE,WAAW,EAAE,CAAC;CACvB,CAAC;AAEF,KAAK,cAAc,GAAG,OAAO,CAAC,MAAM,WAAW,EAAE,QAAQ,CAAC,GAAG,UAAU,CAAC;AAExE,KAAK,sBAAsB,GAAG;IAC5B,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,QAAQ,EAAE,cAAc,CAAC;IACzB,KAAK,EAAE,MAAM,CAAC;IACd,MAAM,CAAC,EAAE,UAAU,CAAC,OAAO,SAAS,CAAC,CAAC;CACvC,CAAC;AAEF,KAAK,sBAAsB,GAAG,sBAAsB,GAAG;IACrD,IAAI,EAAE,MAAM,CAAC;CACd,CAAC;AAEF,KAAK,2BAA2B,GAAG,sBAAsB,GAAG;IAC1D,IAAI,EACA,QAAQ,GACR,QAAQ,GACR,OAAO,GACP,SAAS,GACT,YAAY,GACZ,MAAM,GACN,UAAU,GACV,MAAM,GACN,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,QAAQ,GACR,MAAM,GACN,WAAW,GACX,OAAO,CAAC;CACb,CAAC;AAEF,KAAK,uBAAuB,GAAG,sBAAsB,GAAG;IACtD,IAAI,EAAE,OAAO,CAAC;IACd,OAAO,EAAE,MAAM,EAAE,CAAC;IAElB,KAAK,EAAE,wBAAwB,CAAC;CACjC,CAAC;AAEF,KAAK,0BAA0B,GAAG,sBAAsB,GAAG;IACzD,IAAI,EAAE,UAAU,CAAC;IAEjB,MAAM,EAAE,uBAAuB,CAAC;CACjC,CAAC;AAIF,KAAK,kBAAkB,GACnB,sBAAsB,GACtB,sBAAsB,GACtB,2BAA2B,GAC3B,uBAAuB,GACvB,0BAA0B,CAAC;AAE/B,KAAK,wBAAwB,GAAG;IAC9B,CAAC,SAAS,EAAE,MAAM,GAAG,kBAAkB,CAAC;CACzC,CAAC;AAEF,KAAK,mBAAmB,GAAG;IACzB,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,EAAE,CAAC;IACf,KAAK,EAAE,MAAM,CAAC;IACd,KAAK,EAAE,wBAAwB,CAAC;CACjC,CAAC;AAGF,KAAK,uBAAuB,GAAG;IAC7B,CAAC,SAAS,EAAE,MAAM,GAAG,mBAAmB,CAAC;CAC1C,CAAC;AAEF,KAAK,kBAAkB,GAAG;IACxB,MAAM,EAAE,uBAAuB,CAAC;IAChC,KAAK,EAAE,MAAM,CAAC;CACf,CAAC;AAEF,KAAK,oBAAoB,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,CAAC,EAAE,MAAM,EAAE,KAAK,OAAO,CAAC;AAC3E,KAAK,eAAe,GAAG;IACrB,CAAC,QAAQ,EAAE,MAAM,GAAG,oBAAoB,GAAG,OAAO,CAAC;CACpD,CAAC;AAEF,KAAK,qBAAqB,GAAG;IAC3B,aAAa,EAAE,OAAO,GAAG,eAAe,CAAC;IACzC,WAAW,EAAE,OAAO,CAAC;IACrB,MAAM,CAAC,EAAE,WAAW,CAAC;IACrB,uBAAuB,EAAE,MAAM,CAAC;IAGhC,4BAA4B,CAAC,EAAE,MAAM,CAAC;CACvC,CAAC;AACF,MAAM,MAAM,kBAAkB,GAAG,OAAO,CAAC,qBAAqB,CAAC,CAAC;AA0HhE,MAAM,MAAM,wBAAwB,GAAG;IACrC,QAAQ,EAAE,cAAc,CAAC;CAC1B,CAAC;AACF,MAAM,MAAM,yBAAyB,GAAG,wBAAwB,GAAG;IACjE,QAAQ,EAAE,OAAO,CAAC;IAElB,KAAK,EAAE,oBAAoB,EAAE,CAAC;CAC/B,CAAC;AACF,MAAM,MAAM,4BAA4B,GAAG,wBAAwB,GAAG;IACpE,QAAQ,EAAE,UAAU,CAAC;IAErB,MAAM,EAAE,gBAAgB,CAAC;CAC1B,CAAC;AACF,MAAM,MAAM,oBAAoB,GAC5B,wBAAwB,GACxB,yBAAyB,GACzB,4BAA4B,CAAC;AACjC,MAAM,MAAM,qBAAqB,GAAG;IAClC,KAAK,EAAE,oBAAoB,EAAE,CAAC;CAC/B,CAAC;AACF,MAAM,MAAM,gBAAgB,GAAG;IAC7B,CAAC,SAAS,EAAE,MAAM,GAAG,qBAAqB,CAAC;CAC5C,CAAC;AAoMF,qBAAa,cAAc;IACzB,aAAa,EAAE,eAAe,CAAC;IAC/B,OAAO,EAAE,qBAAqB,CAAC;IAC/B,iBAAiB,SAAK;IACtB,WAAW,SAAK;IAChB,kBAAkB,EAAE,kBAAkB,CAGpC;IAEF,SAAS,UAAQ;IACjB,YAAY,EAAE,MAAM,CAGlB;IAIF,2BAA2B,SAAK;gBAEpB,OAAO,CAAC,EAAE,kBAAkB;IA4BxC,wBAAwB,IAAI,OAAO,CAAC,IAAI,CAAC;IAQzC,kBAAkB;IAUlB,eAAe,CAAC,KAAK,EAAE,SAAS,EAAE,IAAI,EAAE,MAAM,EAAE;IAW1C,UAAU,CAAC,GAAG,EAAE,QAAQ;IAgG9B,SAAS,IAAI,MAAM;IAcnB,cAAc,IAAI,MAAM,EAAE,EAAE;IAO5B,mBAAmB,IAAI,gBAAgB;CAGxC;AAED,wBAAgB,kBAAkB,CAAC,MAAM,EAAE,WAAW,GAAG,WAAW,CASnE;AAED,wBAAsB,0BAA0B,CAC9C,MAAM,EAAE,WAAW,EACnB,OAAO,CAAC,EAAE,kBAAkB,GAC3B,OAAO,CAAC,cAAc,CAAC,CAOzB"}