bio-dts 0.1.0

package/README.md

@@ -0,0 +1,41 @@
+# bio-dts
+
+[![CI](https://github.com/nikku/bio-dts/actions/workflows/CI.yml/badge.svg)](https://github.com/nikku/bio-dts/actions/workflows/CI.yml)
+
+Utilities to generate sane and clean type definitions from JavaScript files.
+
+## About
+
+This module provides `pre` and `post` processing helpers to a type definition pipeline, as well as a simple [generator cli](#usage).
+
+
+## Usage
+
+```sh
+npx bio-dts --outDir dist/types -r lib
+```
+
+## API
+
+```javascript
+import {
+  preTransform,
+  postTransform,
+  generateTypes
+} from 'bio-dts';
+
+// transform JS so it keeps the shape,
+// but is properly digestable by the typescript
+// compiler
+const transformedCode = preTransform(jsCode);
+
+// post process typescript compiler type code
+// removing internals, and fixing up the definitions
+const transformedCode = postTransform(tsCode);
+
+// execute the full pipeline, including invoking the
+// typescript compiler
+generateTypes(files, {
+  outDir: 'dist'
+});
+```

package/bin/bio-dts.js

@@ -0,0 +1,79 @@
+#!/usr/bin/env node
+
+import ts from 'typescript';
+
+import glob from 'tiny-glob';
+
+import path from 'node:path';
+import fs from 'node:fs';
+
+import generateTypes from '../lib/generate-types.js';
+
+async function run() {
+  const args = process.argv.slice(2);
+
+  const {
+    options,
+    fileNames
+  } = ts.parseCommandLine(args);
+
+  const help = args.includes('--help') || args.includes('-h');
+  const verbose = args.includes('--verbose');
+  const recursive = args.includes('--recursive') || args.includes('-r');
+
+  if (help) {
+    console.log(`Usage: bio-dts [options] [...filesOrGlobs]
+
+  Options:
+    --recursive, -r    recurse into directories
+    --verbose          enable verbose logging
+
+    Additional options will be passed to the typescript generator.
+
+  Examples:
+    $ bio-dts --outDir dist -r lib
+    $ bio-dts 'lib/**/*.js'
+`);
+
+    return;
+  }
+
+  const globPattern = recursive ? '**/*.js' : '*.js';
+
+  const files = [];
+
+  for (const fileName of fileNames) {
+
+    try {
+      if (fs.statSync(fileName).isDirectory) {
+
+        const globbedFiles = await glob(globPattern, { cwd: fileName, filesOnly: true });
+
+        files.push(...globbedFiles.map(f => path.join(fileName, f)));
+      } else {
+        files.push(fileName);
+      }
+    } catch (err) {
+
+      // ignore
+    }
+  }
+
+  console.log(`Generating types for ${files.length} files...`);
+
+  generateTypes(files, {
+    allowJs: true,
+    declaration: true,
+    emitDeclarationOnly: true,
+    ...options,
+    verbose
+  });
+
+  console.log('Done.');
+}
+
+run().catch(err => {
+  console.error(err);
+
+  process.exit(1);
+});

package/index.js

@@ -0,0 +1,5 @@
+export { default as preTransform } from './lib/pre-transform.js';
+
+export { default as postTransform } from './lib/post-transform.js';
+
+export { default as generateTypes } from './lib/generate-types.js';

package/lib/generate-types.js

@@ -0,0 +1,62 @@
+import path from 'node:path';
+
+import ts from 'typescript';
+
+import preTransform from './pre-transform.js';
+
+import postTransform from './post-transform.js';
+
+
+/**
+ * @param {string[]} fileNames
+ * @param {ts.CompilerOptions} options
+ */
+export default function generateTypes(fileNames, options) {
+
+  const names = new Set(fileNames.map((p) => path.resolve(p)));
+
+  const host = ts.createCompilerHost(options);
+
+  host._readFile = host.readFile;
+
+  /**
+   * @param {string} fileName
+   *
+   * @return {string}
+   */
+  host.readFile = (fileName) => {
+    options.verbose && console.debug('[generate-types] [pre]', fileName);
+
+    let src = host._readFile(fileName);
+
+    if (names.has(path.resolve(fileName))) {
+      try {
+        src = preTransform(src);
+      } catch (err) {
+        console.error(`failed to parse ${fileName} [pre] with contents ${src}`, err);
+        throw err;
+      }
+    }
+
+    return src;
+  };
+
+  host._writeFile = host.writeFile;
+
+  host.writeFile = (fileName, text, ...args) => {
+    options.verbose && console.debug('[generate-types] [post]', fileName, text);
+
+    try {
+      text = postTransform(text);
+    } catch (err) {
+      console.error(`failed to parse ${fileName} [post] with contents ${text}`, err);
+      throw err;
+    }
+
+    host._writeFile(fileName, text, ...args);
+  };
+
+  const program = ts.createProgram(fileNames, options, host);
+
+  program.emit();
+}

package/lib/parsers/typescript-dts.js

@@ -0,0 +1,15 @@
+import { parser } from 'recast/parsers/babel.js';
+import getBabelOptions from 'recast/parsers/_babel_options.js';
+
+/**
+ * @param {string} source
+ * @param {Overrides} [options]
+ *
+ * @return {import('@babel/types').File}
+ */
+export function parse(source, options) {
+
+  const babelOptions = getBabelOptions.default(options);
+  babelOptions.plugins.push([ 'typescript', { dts: true } ]);
+  return parser.parse(source, babelOptions);
+}

package/lib/parsers/typescript.js

@@ -0,0 +1 @@
+export * from 'recast/parsers/typescript.js';

package/lib/post-transform.js

@@ -0,0 +1,64 @@
+import { print } from 'recast';
+
+import { isArray, isNode, keys, parseDts, path } from './util.js';
+
+
+/**
+ * @param { { name: string, comments?: any[] } }
+ *
+ * @return {boolean}
+ */
+function isPublic(m) {
+
+  const name = m?.key?.name;
+
+  if (!name) {
+    return true;
+  }
+
+  const comments = m?.comments || [];
+
+  return !name.startsWith('_') && !comments.some(c => c.value.includes('@private'));
+}
+
+/**
+ * @param {string} src
+ * @return {string}
+ */
+export default function transform(src) {
+
+  const ast = parseDts(src);
+  const body = path(ast).get('program', 'body');
+
+  traverse(body, function(path) {
+
+    // filter private members
+    if (!isPublic(path.value)) {
+      path.replace();
+    }
+
+    // TODO(nikku): transform illegal typescript methods into override calls
+  });
+
+  return print(ast).code;
+}
+
+function traverse(path, cb) {
+  cb(path);
+
+  if (path.parentPath) {
+    const val = path.value;
+
+    if (!isNode(val) && !isArray(val)) {
+      return;
+    }
+
+    for (const key of keys(val).reverse()) {
+      if ([ 'loc', 'start', 'end' ].includes(key)) {
+        continue;
+      }
+
+      traverse(path.get(key), cb);
+    }
+  }
+}

package/lib/pre-transform.js

@@ -0,0 +1,341 @@
+import { print } from 'recast';
+
+import { matcher, parse, path } from './util.js';
+
+import {
+  builders as b
+} from 'ast-types';
+
+function superCall() {
+  return b.expressionStatement.from({
+    'type': 'ExpressionStatement',
+    'expression': {
+      'type': 'CallExpression',
+      'callee': {
+        'type': 'Super'
+      },
+      'arguments': []
+    }
+  });
+}
+
+function clazzConstructor(cls, comments) {
+  const constructor = cls.ctor.value;
+
+  return b.classMethod.from({
+    'type': 'ClassMethod',
+    'static': false,
+    'key': {
+      'type': 'Identifier',
+      'name': 'constructor'
+    },
+    'kind': 'constructor',
+    'params': constructor.params,
+    'body': constructor.body,
+    'comments': comments
+  });
+}
+
+function clazzDefinition(cls, ctor) {
+
+  const members = [
+    ctor,
+    ...cls.members.filter(
+      m => m.member.type === 'FunctionExpression'
+    ).map(m => {
+      return {
+        'type': 'ClassMethod',
+        'static': false,
+        'key': {
+          'type': 'Identifier',
+          'name': m.name
+        },
+        'kind': 'method',
+        'params': m.member.params,
+        'body': m.member.body,
+        'comments': m.comments
+      };
+    })
+  ];
+
+  const staticMembers = cls.staticMembers.map(m => {
+
+    if (m.member.type === 'FunctionExpression') {
+      return {
+        'type': 'ClassMethod',
+        'static': true,
+        'key': {
+          'type': 'Identifier',
+          'name': m.name
+        },
+        'kind': 'method',
+        'params': m.member.params,
+        'body': m.member.body,
+        'comments': m.comments
+      };
+    }
+
+    return {
+      'type': 'ClassProperty',
+      'static': true,
+      'key': {
+        'type': 'Identifier',
+        'name': m.name
+      },
+      'value': m.member,
+      'comments': m.comments
+    };
+  });
+
+  return b.classDeclaration.from({
+    'type': 'ClassDeclaration',
+    'id': {
+      'type': 'Identifier',
+      'name': cls.name
+    },
+    'superClass': cls.superDeclaration ? {
+      'type': 'Identifier',
+      'name': cls.superDeclaration.clazz
+    } : null,
+    'body': {
+      'type': 'ClassBody',
+      'body': [
+        ...members,
+        ...staticMembers
+      ]
+    }
+  });
+}
+
+function splitComment(cls) {
+
+  const node = cls.node.value;
+
+  const [ comment ] = node.comments?.slice(-1) || [];
+
+  const additionalComments = node.comments?.slice(0, -1) || [];
+
+  const [ intro, ...rest ] = comment?.value?.split(/\* @param/) || [];
+
+  return {
+    ctorComments: rest.length ? [ {
+      leading: true,
+      trailing: false,
+      type: 'CommentBlock',
+      value: '*\n * @param' + rest.join('* @param'),
+    } ] : [],
+    additionalComments: [
+      ...additionalComments,
+      ...(
+        intro ? [
+          {
+            leading: true,
+            type: 'CommentBlock',
+            value: intro
+          }
+        ] : []
+      )
+    ]
+  };
+}
+
+/**
+ * @param {string} src
+ * @return {string}
+ */
+export default function transform(src) {
+
+  const ast = parse(src);
+  const body = path(ast).get('program', 'body');
+
+  const classes = findConstructors(body);
+
+  for (const cls of classes) {
+    const members = cls.members = [
+      ...findEmbeddedMethods(cls),
+      ...findProtoMethods(cls, body)
+    ];
+
+    const staticMembers = cls.staticMembers = [
+      ...findStaticMembers(cls, body)
+    ];
+
+    const superDeclaration = cls.superDeclaration = findSuperDeclarations(cls, body);
+
+    if (superDeclaration) {
+      cls.ctor.get('body', 'body').unshift(superCall());
+
+      superDeclaration.superCalls.forEach(m => m.node.replace());
+    }
+
+    const {
+      ctorComments,
+      additionalComments
+    } = splitComment(cls);
+
+    const ctor = clazzConstructor(cls, ctorComments);
+
+    cls.ctor.replace(clazzDefinition(cls, ctor));
+    cls.node.get('comments').replace(additionalComments);
+
+    members.forEach(m => {
+      if (m.member.type === 'FunctionExpression') {
+        m.node.replace();
+      }
+    });
+
+    staticMembers.forEach(m => {
+      m.node.replace();
+    });
+
+    if (superDeclaration) {
+      superDeclaration.importDeclarations.forEach(m => m.node.replace());
+      superDeclaration.inheritsExpression.node.replace();
+    }
+  }
+
+  return print(ast).code;
+}
+
+function findSuperDeclarations(cls, nodes) {
+
+  const importsMatches = matcher`
+    import $1 from 'inherits-browser';
+  `;
+
+  const importDeclarations = importsMatches(nodes);
+
+  for (const [ _, identifier ] of importDeclarations) {
+
+    const inheritsMatches = matcher`
+      ${identifier.value.name}(${cls.name}, $1);
+    `;
+
+    const inheritsExpressions = inheritsMatches(nodes);
+
+    if (inheritsExpressions.length > 1) {
+      throw new Error('more than one <inherits> declaration');
+    }
+
+    if (inheritsExpressions.length === 1) {
+
+      const clazz = inheritsExpressions[0][1].value.name;
+
+      const inheritsExpression = {
+        node: inheritsExpressions[0][0]
+      };
+
+      const superCallMatches = matcher`
+        ${clazz}.call($$$);
+      `;
+
+      const superCalls = superCallMatches(cls.ctor.get('body', 'body'));
+
+      return {
+        importDeclarations: importDeclarations.map(m => ({ node: m[0] })),
+        superCalls: superCalls.map(m => ({ node: m[0] })),
+        clazz,
+        inheritsExpression
+      };
+    }
+  }
+
+}
+
+function findConstructors(nodes) {
+
+  const fns = matcher`
+    function $1($$$) {
+      $$$
+    }
+  `;
+
+  const namedExportFns = matcher`
+    export function $1($$$) {
+      $$$
+    }
+  `;
+
+  const defaultExportFns = matcher`
+    export default function $1($$$) {
+      $$$
+    }
+  `;
+
+  return [
+    ...fns(nodes),
+    ...namedExportFns(nodes).map(
+      m => [ m[0].get('declaration'), m[1], m[0] ]
+    ),
+    ...defaultExportFns(nodes).map(
+      m => [ m[0].get('declaration'), m[1], m[0] ]
+    )
+  ].map(match => {
+
+    const [ ctor, identifier, node = ctor ] = match;
+
+    const name = identifier.value.name;
+
+    if (/^[A-Z]/.test(name)) {
+      return {
+        name,
+        ctor,
+        node
+      };
+    }
+  }).filter(n => n);
+}
+
+function findStaticMembers(cls, nodes) {
+
+  const names = matcher`
+    ${cls.name}.$1 = $2;
+  `;
+
+  return names(nodes).map(match => {
+    const [ node, identifier, member ] = match;
+
+    return {
+      name: identifier.value.name,
+      member: member.value,
+      comments: node.get('comments').value,
+      node
+    };
+  });
+}
+
+function findProtoMethods(cls, nodes) {
+
+  const names = matcher`
+    ${cls.name}.prototype.$1 = $2;
+  `;
+
+  return names(nodes).map(match => {
+    const [ node, identifier, member ] = match;
+
+    return {
+      name: identifier.value.name,
+      member: member.value,
+      comments: node.get('comments').value,
+      node
+    };
+  });
+}
+
+function findEmbeddedMethods(cls) {
+
+  const memberDeclarations = matcher`
+    this.$1 = $2;
+  `;
+
+  return memberDeclarations(cls.ctor.get('body', 'body')).map(match => {
+    const [ node, identifier, member ] = match;
+
+    return {
+      name: identifier.value.name,
+      member: member.value,
+      comments: node.get('comments').value,
+      node
+    };
+  });
+}

package/lib/util.js

@@ -0,0 +1,262 @@
+import { parse as recastParse } from 'recast';
+import { Path as PathConstructor } from 'ast-types';
+
+/**
+ * @typedef { {
+ *   get(...any) => any,
+ *   value: any
+ * } } Path
+ */
+
+import * as typescriptParser from './parsers/typescript.js';
+import * as typescriptDtsParser from './parsers/typescript-dts.js';
+
+const DBG = /match/.test(process?.env?.LOG_DEBUG);
+
+/**
+ * Return numeric or string keys of the given object.
+ *
+ * @param {any} obj
+ * @return {(string|number)[]}
+ */
+export function keys(obj) {
+  return Object.keys(obj).map(k => {
+    const n = parseInt(k, 10);
+
+    return isNaN(n) ? k : n;
+  });
+}
+
+/**
+ * @param {any} arr
+ *
+ * @return {boolean}
+ */
+export function isArray(arr) {
+  return Array.isArray(arr);
+}
+
+export function isNode(obj) {
+  return Object.prototype.toString.call(obj) === '[object Object]';
+}
+
+export function hasProperty(obj, property) {
+  return Object.prototype.hasOwnProperty.call(obj, property);
+}
+
+/**
+ * @param {string} code
+ *
+ * @return {any}
+ */
+export function parse(code) {
+  return recastParse(code, {
+    parser: typescriptParser
+  });
+}
+
+export function parseDts(code) {
+  return recastParse(code, {
+    parser: typescriptDtsParser
+  });
+}
+
+/**
+ * @param {any} node
+ *
+ * @return {Path}
+ */
+export function path(node) {
+  return new PathConstructor(node);
+}
+
+/**
+ * @param {string[]} strings
+ * @param {...any} args
+ *
+ * @return { (node) => [][] }
+ */
+export function matcher(strings, ...args) {
+
+  const code = strings.reduce((code, str, i) => {
+
+    code += str;
+
+    if (args[i]) {
+      code += args[i];
+    }
+
+    return code;
+  }, '');
+
+  const ast = parse(code, {
+    parser: typescriptParser
+  });
+
+  const body = path(ast).get('program', 'body');
+
+  if (body.value.length !== 1) {
+    throw new Error(`${ code } is more than one statement!`);
+  }
+
+  const expr = body.get(0);
+
+  /**
+   * Match for wildcards:
+   *
+   *   - block statement with a single $$$
+   *   - expression statement with $$$
+   *   - identifier $$$
+   *   - wrapped as list
+   *
+   * @param {any} node
+   *
+   * @return {boolean}
+   */
+  function wildcard(node) {
+
+    if (isArray(node) && node.length === 1) {
+      node = node[0];
+    }
+
+    if (node && node.type === 'ExpressionStatement') {
+      node = node.expression;
+    }
+
+    if (node && node.type === 'Identifier' && node.name === '$$$') {
+      return true;
+    }
+
+  }
+
+  /**
+   * Matches ${IDENTIFIER} single node.
+   *
+   * @param {any} node
+   *
+   * @return {any|null}
+   */
+  function ident(node) {
+    if (node.type === 'Identifier' && /^\$.+/.test(node.name)) {
+      return node.name.substring(1);
+    }
+
+    return null;
+  }
+
+  /**
+   * Matches $${IDENTIFIER} many nodes:
+   *
+   *   - block statement with a single $${IDENT}
+   *   - expression statement with $${IDENT}
+   *   - identifier $${IDENT}
+   *   - wrapped as list
+   *
+   * @param {any} node
+   *
+   * @return {any|null}
+   */
+  function anyIdent(node) {
+
+    if (isArray(node) && node.length === 1) {
+      node = node[0];
+    }
+
+    if (node && node.type === 'ExpressionStatement') {
+      node = node.expression;
+    }
+
+    if (node && node.type === 'Identifier' && /^\$\$.+/.test(node.name)) {
+      return node.name.substring(2);
+    }
+
+    return null;
+  }
+
+  /**
+   * @param {Path} node
+   * @param {Path} expectedNode
+   * @param {Path[]} results
+   *
+   * @return {Path[] || null}
+   */
+  function matchNode(node, expectedNode, results = []) {
+
+    if (!node.value) {
+      DBG && console.log('BAIL no value');
+      return;
+    }
+
+    const id = ident(expectedNode.value);
+
+    if (id) {
+      results[id] = node;
+
+      return results;
+    }
+
+    for (const key of keys(expectedNode.value)) {
+      if ([ 'loc', 'start', 'end' ].includes(key)) {
+        continue;
+      }
+
+      if (!hasProperty(node.value, key)) {
+        DBG && console.log('BAIL no property', key);
+        return;
+      }
+
+      const expectedVal = expectedNode.get(key);
+      const nodeVal = node.get(key);
+
+      if (wildcard(expectedVal.value)) {
+        continue;
+      }
+
+      const id = anyIdent(expectedVal.value);
+
+      if (id) {
+        results[id] = nodeVal;
+        continue;
+      }
+
+      if (isArray(expectedVal.value)) {
+        if (
+          expectedVal.value.some((el, idx) => {
+            return !matchNode(nodeVal.get(idx), expectedVal.get(idx), results);
+          })
+        ) {
+          DBG && console.log('BAIL no array match', key);
+          return;
+        }
+      } else
+      if (isNode(expectedVal.value)) {
+        if (!matchNode(nodeVal, expectedVal, results)) {
+          DBG && console.log('BAIL no match', key);
+          return;
+        }
+      } else {
+        if (nodeVal.value !== expectedVal.value) {
+          DBG && console.log('BAIL no primitive match', key);
+          return;
+        }
+      }
+    }
+
+    return results;
+  }
+
+  return function match(nodes) {
+    const matches = [];
+
+    for (const key of keys(nodes.value)) {
+      const node = nodes.get(key);
+      const matched = matchNode(node, expr);
+
+      if (matched) {
+        matches.push([ node, ...matched.slice(1) ]);
+      }
+    }
+
+    return matches;
+  };
+}

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "bio-dts",
+  "version": "0.1.0",
+  "description": "Generate sane and clean types from JavaScript sources",
+  "type": "module",
+  "main": "index.js",
+  "bin": "bin/bio-dts",
+  "scripts": {
+    "all": "npm run lint && npm run test && npm run test:dts",
+    "lint": "eslint .",
+    "test": "mocha test",
+    "test:dts": "node bin/bio-dts.js --outDir dist -r test/fixtures"
+  },
+  "author": "Nico Rehwaldt",
+  "license": "MIT",
+  "dependencies": {
+    "@babel/parser": "^7.21.2",
+    "recast": "^0.22.0",
+    "tiny-glob": "^0.2.9",
+    "typescript": "^4.9.5"
+  },
+  "devDependencies": {
+    "chai": "^4.3.7",
+    "eslint": "^8.36.0",
+    "eslint-plugin-bpmn-io": "^1.0.0",
+    "mocha": "^10.2.0"
+  },
+  "files": [
+    "bin",
+    "lib",
+    "index.js"
+  ]
+}