@thi.ng/tangle 0.1.91 → 0.2.1

package/CHANGELOG.md

@@ -1,6 +1,6 @@
 # Change Log
 
-- **Last updated**: 2024-03-01T15:22:50Z
+- **Last updated**: 2024-03-07T20:40:47Z
 - **Generator**: [thi.ng/monopub](https://thi.ng/monopub)
 
 All notable changes to this project will be documented in this file.
@@ -9,6 +9,20 @@ See [Conventional Commits](https://conventionalcommits.org/) for commit guidelin
 **Note:** Unlisted _patch_ versions only involve non-code or otherwise excluded changes
 and/or version bumps of transitive dependencies.
 
+## [0.2.0](https://github.com/thi-ng/umbrella/tree/@thi.ng/tangle@0.2.0) (2024-03-07)
+
+#### 🚀 Features
+
+- add support for code blocks in JS/TS docstrings ([5b68f19](https://github.com/thi-ng/umbrella/commit/5b68f19))
+  - update CodeBlockFormat, allow regexps as prefix/suffix
+    - add optional body xform fn
+  - add ".js", ".ts" as supported file extensions
+  - update matching logic in extractBlocks()
+
+#### 🩹 Bug fixes
+
+- off-by-one error in extractBlocks() ([d7d7e03](https://github.com/thi-ng/umbrella/commit/d7d7e03))
+
 ### [0.1.86](https://github.com/thi-ng/umbrella/tree/@thi.ng/tangle@0.1.86) (2024-02-22)
 
 #### ♻️ Refactoring

package/README.md

@@ -48,12 +48,18 @@ introduction. For all LP projects, a so-called "tangling" step is required to
 produce workable, standard source code files from these documents. This package
 provides just that:
 
-Extract, expand, transclude, combine and assemble code blocks from Markdown or
-[Org-mode](https://orgmode.org) files into actual/traditional source files. A
-single LP source file can contain code for multiple languages. Each code block
-can define its target file and include [noweb-style
+Extract, expand, transclude, combine and assemble code blocks from arbitrary text files into actual/traditional source code files. The following input formats are supported:
+
+- Markdown
+- Markdown codeblocks embedded inside docstrings of JavaScript/TypeScript source files
+- [Org-mode](https://orgmode.org)
+
+A single input file can contain code for multiple languages. Each individual
+code block can define its own target file and include [noweb-style
 references](https://orgmode.org/manual/Noweb-Reference-Syntax.html) to other
-code blocks, either from the same or even from other files.
+code blocks, either from the same or even from other files. If multiple code
+blocks in the same input file reference the same output file, they will be
+concatenated.
 
 The package provides both a basic API and a CLI wrapper to perform the
 "tangling" tasks (an expression borrowed from
@@ -326,7 +332,7 @@ For Node.js REPL:
 const tangle = await import("@thi.ng/tangle");
 ```
 
-Package sizes (brotli'd, pre-treeshake): ESM: 1.83 KB
+Package sizes (brotli'd, pre-treeshake): ESM: 1.99 KB
 
 ## Dependencies
 

package/api.d.ts

@@ -1,4 +1,4 @@
-import type { Fn2, FnAnyT, IObjectOf, Predicate } from "@thi.ng/api";
+import type { Fn, Fn2, FnAnyT, IObjectOf, Predicate } from "@thi.ng/api";
 import { type ILogger } from "@thi.ng/logger";
 export interface Block {
     id: string;
@@ -59,8 +59,24 @@ export interface TangleOpts {
     comments: boolean;
 }
 export interface CodeBlockFormat {
-    prefix: string;
-    suffix: string;
+    /**
+     * String or regexp matching the beginning of a code block. If given a
+     * regexp, it will become part of a larger one and any regexp flags given
+     * here will be ignored. Also the given regexp pattern will always be
+     * prefixed with the line-start symbol (`^`), so it must not be given
+     * here...
+     */
+    prefix: string | RegExp;
+    /**
+     * String or regexp matching the end of a code block. If given a
+     * regexp, it MUST have the multiline flag enabled!
+     */
+    suffix: string | RegExp;
+    /**
+     * Optional function to transform the codeblock's body (e.g. to remove any
+     * prefix characters or remove trailing whitespace).
+     */
+    xform?: Fn<string, string>;
 }
 export declare const BLOCK_FORMATS: IObjectOf<CodeBlockFormat>;
 export declare const COMMENT_FORMATS: IObjectOf<string | [string, string]>;

package/api.js

@@ -1,4 +1,15 @@
 import { ConsoleLogger, LogLevel, ROOT } from "@thi.ng/logger";
+import { split } from "@thi.ng/strings";
+import { map, str, transduce } from "@thi.ng/transducers";
+const JS_DOC = {
+  prefix: /^\s*\*\s+```/,
+  suffix: /^\s*\*\s+```/m,
+  xform: (body) => transduce(
+    map((x) => x.replace(/^\s*\*\s?/, "")),
+    str("\n"),
+    split(body)
+  )
+};
 const BLOCK_FORMATS = {
   ".md": {
     prefix: "```",
@@ -7,7 +18,9 @@ const BLOCK_FORMATS = {
   ".org": {
     prefix: "#+BEGIN_SRC ",
     suffix: "#+END_SRC"
-  }
+  },
+  ".js": JS_DOC,
+  ".ts": JS_DOC
 };
 const C = "//";
 const P = "#";

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@thi.ng/tangle",
-  "version": "0.1.91",
+  "version": "0.2.1",
   "description": "Literate programming code block tangling / codegen utility, inspired by org-mode & noweb",
   "type": "module",
   "module": "./index.js",
@@ -33,23 +33,24 @@
     "doc:ae": "mkdir -p .ae/doc .ae/temp && api-extractor run --local --verbose",
     "doc:readme": "bun ../../tools/src/module-stats.ts && bun ../../tools/src/readme.ts",
     "pub": "yarn npm publish --access public",
-    "test": "testament test"
+    "test": "testament test",
+    "tool:tangle": "../../node_modules/.bin/tangle src/**/*.ts"
   },
   "dependencies": {
-    "@thi.ng/api": "^8.9.27",
-    "@thi.ng/args": "^2.3.20",
-    "@thi.ng/checks": "^3.5.1",
-    "@thi.ng/compare": "^2.2.23",
-    "@thi.ng/date": "^2.7.4",
-    "@thi.ng/errors": "^2.4.19",
-    "@thi.ng/file-io": "^1.3.5",
-    "@thi.ng/logger": "^3.0.4",
-    "@thi.ng/strings": "^3.7.20",
-    "@thi.ng/transducers": "^8.9.9"
+    "@thi.ng/api": "^8.9.29",
+    "@thi.ng/args": "^2.3.22",
+    "@thi.ng/checks": "^3.5.2",
+    "@thi.ng/compare": "^2.2.25",
+    "@thi.ng/date": "^2.7.6",
+    "@thi.ng/errors": "^2.4.20",
+    "@thi.ng/file-io": "^1.3.7",
+    "@thi.ng/logger": "^3.0.5",
+    "@thi.ng/strings": "^3.7.22",
+    "@thi.ng/transducers": "^8.9.11"
   },
   "devDependencies": {
     "@microsoft/api-extractor": "^7.40.1",
-    "@thi.ng/testament": "^0.4.21",
+    "@thi.ng/testament": "^0.4.22",
     "esbuild": "^0.20.0",
     "rimraf": "^5.0.5",
     "typedoc": "^0.25.7",
@@ -93,5 +94,5 @@
     "status": "alpha",
     "year": 2022
   },
-  "gitHead": "df9e312af741d87e6b450afcfea6a6e381662b1e\n"
+  "gitHead": "69100942474942f7446ac645d59d91e7dfc352f9\n"
 }

package/tangle.js

@@ -1,8 +1,7 @@
 import { isPlainObject, isString } from "@thi.ng/checks";
 import { compareByKey } from "@thi.ng/compare";
 import { FMT_ISO_SHORT } from "@thi.ng/date";
-import { defError } from "@thi.ng/errors/deferror";
-import { illegalArgs } from "@thi.ng/errors/illegal-arguments";
+import { defError, illegalArgs, illegalState } from "@thi.ng/errors";
 import { readText } from "@thi.ng/file-io";
 import { split } from "@thi.ng/strings";
 import { assocObj, map, transduce } from "@thi.ng/transducers";
@@ -19,38 +18,51 @@ const UnknownBlockError = defError(
 const extractBlocks = (src, { format, logger }) => {
   let nextID = 0;
   const blocks = {};
-  const re = new RegExp(
-    `^${format.prefix.replace("+", "\\+")}(\\w+)\\s+(.+)$`,
+  const prefix = new RegExp(
+    (isString(format.prefix) ? `^${format.prefix.replace("+", "\\+")}` : format.prefix.source) + `(\\w+)\\s+(.+)$`,
     "gm"
   );
-  let match;
-  while (match = re.exec(src)) {
-    let { id, tangle, noweb, publish } = parseBlockHeader(match[2]);
+  const suffix = isString(format.suffix) ? new RegExp(`${format.suffix.replace("+", "\\+")}`) : format.suffix;
+  let matchPrefix;
+  while (matchPrefix = prefix.exec(src)) {
+    let { id, tangle, noweb, publish } = parseBlockHeader(matchPrefix[2]);
     !id && (id = `__block-${nextID++}`);
-    const matchStart = match.index;
-    const start = src.indexOf("\n", matchStart) + 1;
-    const end = src.indexOf(`
-${format.suffix}`, matchStart + 1);
+    const matchStart = matchPrefix.index;
+    const start = src.indexOf("\n", matchStart + 1) + 1;
     logger.debug(
-      "codeblock",
+      "codeblock ID:",
       id,
-      start,
-      end,
+      "matchStart:",
       matchStart,
-      src.substring(start, 8)
+      "start:",
+      start
+    );
+    const matchSuffix = suffix.exec(src.substring(start));
+    if (!matchSuffix)
+      illegalState("no codeblock end found");
+    const end = start + matchSuffix.index;
+    const matchEnd = end + matchSuffix[0].length + 1;
+    logger.debug(
+      "codeblock ID:",
+      id,
+      "end:",
+      end,
+      "matchEnd:",
+      matchEnd,
+      matchSuffix[0]
     );
-    const body = src.substring(start, end);
+    const body = src.substring(start, end - 1);
     blocks[id] = {
       id,
-      lang: match[1],
+      lang: matchPrefix[1],
       tangle,
       publish,
       noweb,
       start,
       end,
       matchStart,
-      matchEnd: end + format.suffix.length + 2,
-      body
+      matchEnd,
+      body: format.xform ? format.xform(body) : body
     };
   }
   return blocks;