@panagos/jojoscript 0.7.0

package/src/compiler.js

@@ -0,0 +1,169 @@
+import fs from 'node:fs'
+import path from 'node:path'
+
+import {
+  transformPipelines,
+  findPipelineStdlibFunctions
+} from './pipeline-parser.js'
+
+import {
+  transformMutableDeclarations,
+  transformConstDeclarations,
+  transformMultiBindingDeclarations,
+  transformDestructuringDeclarations,
+  transformFunctionDeclarations,
+  transformExpressionFunctions,
+  transformLocalImports
+} from './transforms.js'
+
+import { transformMatchExpressions } from './match-parser.js'
+import { transformTypeAnnotations } from './type-annotations.js'
+
+const JOJO_RUNTIME = 'jojoscript/runtime'
+
+export function compile(source) {
+  let result = source
+
+  /*
+   * Optional type annotations, stripped and re-expressed as JSDoc before
+   * anything else touches `fn` declarations.
+   */
+  result = transformTypeAnnotations(result)
+
+  /*
+   * Jojo functions.
+   */
+  result = transformExpressionFunctions(result)
+  result = transformFunctionDeclarations(result)
+
+  /*
+   * `match` expressions.
+   */
+  result = transformMatchExpressions(result)
+
+  /*
+   * Jojo declarations.
+   */
+  result = transformMultiBindingDeclarations(result)
+  result = transformDestructuringDeclarations(result)
+  result = transformMutableDeclarations(result)
+  result = transformConstDeclarations(result)
+
+  /*
+   * Remember which pipeline stages need the Jojo runtime before
+   * the pipeline syntax itself is removed.
+   */
+  const pipelineStdlibFunctions =
+    findPipelineStdlibFunctions(result)
+
+  /*
+   * Pipelines.
+   */
+  result = transformPipelines(result)
+
+  /*
+   * ES module imports.
+   */
+  result = transformLocalImports(result)
+
+  /*
+   * Generated JavaScript imports the runtime as a normal package.
+   * This makes compiled applications portable and avoids globals. If the
+   * source already has a manual `jojoscript/runtime` import (e.g. for a
+   * stdlib function that is only ever called directly, never through a
+   * pipeline stage), the auto-imported names are merged into it instead
+   * of creating a second, colliding import declaration.
+   */
+  if (pipelineStdlibFunctions.length) {
+    result = mergeRuntimeImport(result, pipelineStdlibFunctions)
+  }
+
+  return result.endsWith('\n') ? result : result + '\n'
+}
+
+const RUNTIME_IMPORT_PATTERN = /import\s*\{([^}]*)\}\s*from\s*(["'])jojoscript\/runtime\2/
+
+function mergeRuntimeImport(source, stdlibFunctions) {
+  const existing = source.match(RUNTIME_IMPORT_PATTERN)
+
+  if (!existing) {
+    return `import { ${stdlibFunctions.join(', ')} } from "${JOJO_RUNTIME}"\n` + source
+  }
+
+  const existingNames = existing[1].split(',').map(name => name.trim()).filter(Boolean)
+  const merged = [...new Set([...existingNames, ...stdlibFunctions])]
+
+  return source.replace(
+    RUNTIME_IMPORT_PATTERN,
+    `import { ${merged.join(', ')} } from "${JOJO_RUNTIME}"`
+  )
+}
+
+export function compileFile(input, outDir = null) {
+  const src = path.resolve(input)
+  const dir = outDir
+    ? path.resolve(outDir)
+    : path.join(path.dirname(src), 'dist')
+
+  fs.mkdirSync(dir, { recursive: true })
+
+  const out = path.join(
+    dir,
+    path.basename(src, path.extname(src)) + '.js'
+  )
+
+  fs.writeFileSync(
+    out,
+    compile(fs.readFileSync(src, 'utf8')),
+    'utf8'
+  )
+
+  return { input: src, output: out }
+}
+
+export function compileDirectory(inputDir, outDir = null) {
+  const root = path.resolve(inputDir)
+  const target = outDir
+    ? path.resolve(outDir)
+    : path.join(root, 'dist')
+
+  const result = []
+
+  function walk(dir) {
+    for (const entry of fs.readdirSync(dir, { withFileTypes: true })) {
+      if (
+        entry.name === 'node_modules' ||
+        entry.name === 'dist' ||
+        entry.name.startsWith('.')
+      ) {
+        continue
+      }
+
+      const full = path.join(dir, entry.name)
+
+      if (entry.isDirectory()) {
+        walk(full)
+        continue
+      }
+
+      if (!entry.name.endsWith('.jojo')) continue
+
+      const rel =
+        path.relative(root, full).slice(0, -'.jojo'.length) + '.js'
+
+      const out = path.join(target, rel)
+
+      fs.mkdirSync(path.dirname(out), { recursive: true })
+      fs.writeFileSync(
+        out,
+        compile(fs.readFileSync(full, 'utf8')),
+        'utf8'
+      )
+
+      result.push({ input: full, output: out })
+    }
+  }
+
+  walk(root)
+  return result
+}

package/src/index.js

@@ -0,0 +1,19 @@
+export {
+  compile,
+  compileFile,
+  compileDirectory
+} from './compiler.js'
+
+export {
+  transformPipelines,
+  emitPipeline,
+  findPipelineStdlibFunctions
+} from './pipeline-parser.js'
+
+export {
+  transformMatchExpressions
+} from './match-parser.js'
+
+export {
+  transformTypeAnnotations
+} from './type-annotations.js'

package/src/lexer.js

@@ -0,0 +1,178 @@
+export function tokenize(source) {
+  const tokens = []
+  let i = 0
+
+  while (i < source.length) {
+    const start = i
+    const c = source[i]
+
+    if (/\s/.test(c)) {
+      i++
+      continue
+    }
+
+    if (source.startsWith('//', i)) {
+      i += 2
+      while (i < source.length && source[i] !== '\n') i++
+      continue
+    }
+
+    if (source.startsWith('/*', i)) {
+      i += 2
+      while (i < source.length && !source.startsWith('*/', i)) i++
+      if (i < source.length) i += 2
+      continue
+    }
+
+    if (c === '"' || c === "'" || c === '`') {
+      const quote = c
+      i++
+      let templateExpressionDepth = 0
+      while (i < source.length) {
+        if (source[i] === '\\') {
+          i += 2
+          continue
+        }
+        if (quote === '`' && source.startsWith('${', i)) {
+          templateExpressionDepth++
+          i += 2
+          continue
+        }
+        if (quote === '`' && source[i] === '}' && templateExpressionDepth > 0) {
+          templateExpressionDepth--
+          i++
+          continue
+        }
+        if (source[i] === quote && templateExpressionDepth === 0) {
+          i++
+          break
+        }
+        i++
+      }
+      tokens.push({ type: 'string', value: source.slice(start, i), start, end: i })
+      continue
+    }
+
+    if (/[A-Za-z_$]/.test(c)) {
+      i++
+      while (i < source.length && /[A-Za-z0-9_$]/.test(source[i])) i++
+      tokens.push({ type: 'identifier', value: source.slice(start, i), start, end: i })
+      continue
+    }
+
+    if (/[0-9]/.test(c)) {
+      i++
+      while (i < source.length && /[A-Za-z0-9_.$]/.test(source[i])) i++
+      tokens.push({ type: 'number', value: source.slice(start, i), start, end: i })
+      continue
+    }
+
+    const operators = [
+      '>>>=' , '===', '!==', '**=', '&&=', '||=', '??=',
+      '=>', '|>', '?.', '??', '==', '!=', '<=', '>=',
+      '&&', '||', '**', '++', '--', '+=', '-=', '*=', '/=',
+      '%=', '...', '<<', '>>', '>>>', '?.'
+    ]
+
+    const op = operators.find(value => source.startsWith(value, i))
+    if (op) {
+      tokens.push({
+        type: op === '|>' ? 'pipeline' : 'operator',
+        value: op,
+        start,
+        end: i + op.length
+      })
+      i += op.length
+      continue
+    }
+
+    tokens.push({ type: 'punctuation', value: c, start, end: i + 1 })
+    i++
+  }
+
+  return tokens
+}
+
+export function significantTokens(tokens) {
+  return tokens
+}
+
+/*
+ * Produces a same-length copy of `source` where the contents of comments
+ * and/or string/template literals are replaced with blank spaces (newlines
+ * are preserved so line-based/anchored regexes keep working).
+ *
+ * This lets regex-based transforms and character scanners check "is this
+ * position real code?" by comparing against the mask, without having to
+ * duplicate string/comment-parsing logic everywhere.
+ */
+export function maskNonCode(source, options = {}) {
+  const maskStrings = options.strings !== false
+  const maskComments = options.comments !== false
+
+  let mask = ''
+  let i = 0
+
+  while (i < source.length) {
+    if (source.startsWith('//', i)) {
+      const start = i
+      while (i < source.length && source[i] !== '\n') i++
+      mask += maskComments ? blank(source, start, i) : source.slice(start, i)
+      continue
+    }
+
+    if (source.startsWith('/*', i)) {
+      const start = i
+      i += 2
+      while (i < source.length && !source.startsWith('*/', i)) i++
+      if (i < source.length) i += 2
+      mask += maskComments ? blank(source, start, i) : source.slice(start, i)
+      continue
+    }
+
+    const c = source[i]
+
+    if (c === '"' || c === "'" || c === '`') {
+      const quote = c
+      const start = i
+      i++
+      let templateExpressionDepth = 0
+      while (i < source.length) {
+        if (source[i] === '\\') {
+          i += 2
+          continue
+        }
+        if (quote === '`' && source.startsWith('${', i)) {
+          templateExpressionDepth++
+          i += 2
+          continue
+        }
+        if (quote === '`' && source[i] === '}' && templateExpressionDepth > 0) {
+          templateExpressionDepth--
+          i++
+          continue
+        }
+        if (source[i] === quote && templateExpressionDepth === 0) {
+          i++
+          break
+        }
+        i++
+      }
+      mask += maskStrings ? blank(source, start, i) : source.slice(start, i)
+      continue
+    }
+
+    mask += c
+    i++
+  }
+
+  return mask
+}
+
+function blank(source, start, end) {
+  let result = ''
+  for (let i = start; i < end; i++) {
+    result += source[i] === '\n' ? '\n' : ' '
+  }
+  return result
+}

package/src/match-parser.js

@@ -0,0 +1,215 @@
+import { tokenize, significantTokens } from './lexer.js'
+
+/*
+ * `match` compiles a value-equality expression construct:
+ *
+ *   match status {
+ *     "ok" => handleOk(),
+ *     "error" => handleError(),
+ *     _ => handleUnknown()
+ *   }
+ *
+ * into an immediately-invoked arrow function that compares the subject
+ * against each pattern with `===`, in order, falling through to the
+ * mandatory default `_` arm:
+ *
+ *   (() => {
+ *     const __match = status;
+ *     if (__match === ("ok")) return handleOk();
+ *     if (__match === ("error")) return handleError();
+ *     return handleUnknown();
+ *   })()
+ *
+ * This is deliberately a value-equality match, not structural pattern
+ * matching: patterns are plain expressions compared with `===`, and arms
+ * are single expressions (no statement blocks). The default `_` arm is
+ * required and must be last.
+ */
+
+function text(source, tokens) {
+  if (!tokens.length) return ''
+  return source.slice(tokens[0].start, tokens[tokens.length - 1].end)
+}
+
+function splitTopLevelCommas(tokens) {
+  const result = []
+  let start = 0
+  let depth = 0
+
+  for (let i = 0; i < tokens.length; i++) {
+    const value = tokens[i].value
+    if (value === '(' || value === '[' || value === '{') depth++
+    else if (value === ')' || value === ']' || value === '}') depth--
+    else if (value === ',' && depth === 0) {
+      result.push(tokens.slice(start, i))
+      start = i + 1
+    }
+  }
+
+  if (start < tokens.length) result.push(tokens.slice(start))
+  return result
+}
+
+/*
+ * Distinguishes the `match` keyword from ordinary identifiers named
+ * "match" (e.g. `string.match(regex)`, `const match = ...`, a `match`
+ * loop variable, or a `match:` object key).
+ */
+function isMatchKeyword(tokens, index) {
+  const token = tokens[index]
+  if (token.type !== 'identifier' || token.value !== 'match') return false
+
+  const prev = tokens[index - 1]
+  if (prev && (prev.value === '.' || prev.value === 'function' || prev.value === 'class')) {
+    return false
+  }
+
+  const next = tokens[index + 1]
+  if (!next) return false
+  if (['(', ':', '=', '.', ',', ')', ']', '}'].includes(next.value)) return false
+
+  return true
+}
+
+/*
+ * Scans forward from a candidate `match` keyword looking for the subject
+ * expression followed by a top-level `{`. Bounded defensively: if the
+ * scan leaves the enclosing expression (unbalanced closing bracket) or
+ * runs too far without finding `{`, this is not a match construct.
+ */
+function findMatchBlock(source, tokens, index) {
+  let depth = 0
+  let braceOpen = -1
+
+  const limit = Math.min(tokens.length, index + 60)
+
+  for (let i = index + 1; i < limit; i++) {
+    const value = tokens[i].value
+    const prevToken = tokens[i - 1]
+
+    if (prevToken) {
+      const gap = source.slice(prevToken.end, tokens[i].start)
+      if (depth === 0 && (gap.match(/\n/g) || []).length > 1) return null
+    }
+
+    if (value === '(' || value === '[') { depth++; continue }
+    if (value === ')' || value === ']') {
+      depth--
+      if (depth < 0) return null
+      continue
+    }
+    if (value === ';' && depth === 0) return null
+    if (value === '{' && depth === 0) { braceOpen = i; break }
+  }
+
+  if (braceOpen === -1) return null
+
+  const subjectTokens = tokens.slice(index + 1, braceOpen)
+  if (!subjectTokens.length) return null
+
+  let braceDepth = 0
+  let braceClose = -1
+
+  for (let j = braceOpen; j < tokens.length; j++) {
+    if (tokens[j].value === '{') braceDepth++
+    else if (tokens[j].value === '}') {
+      braceDepth--
+      if (braceDepth === 0) { braceClose = j; break }
+    }
+  }
+
+  if (braceClose === -1) {
+    throw new SyntaxError(`Unclosed match block:\n${text(source, tokens.slice(index, braceOpen + 1))}`)
+  }
+
+  return { subjectTokens, braceOpen, braceClose }
+}
+
+function parseArms(source, tokens) {
+  const groups = splitTopLevelCommas(tokens).filter(group => group.length)
+
+  if (!groups.length) {
+    throw new SyntaxError('match requires at least one arm')
+  }
+
+  return groups.map(group => {
+    let depth = 0
+    let arrowIndex = -1
+
+    for (let i = 0; i < group.length; i++) {
+      const value = group[i].value
+      if (value === '(' || value === '[' || value === '{') depth++
+      else if (value === ')' || value === ']' || value === '}') depth--
+      else if (value === '=>' && depth === 0) { arrowIndex = i; break }
+    }
+
+    if (arrowIndex === -1) {
+      throw new SyntaxError(`Invalid match arm, expected \`pattern => expression\`:\n${text(source, group)}`)
+    }
+
+    const pattern = text(source, group.slice(0, arrowIndex)).trim()
+    const expression = text(source, group.slice(arrowIndex + 1)).trim()
+
+    if (!pattern || !expression) {
+      throw new SyntaxError(`Invalid match arm, expected \`pattern => expression\`:\n${text(source, group)}`)
+    }
+
+    return { pattern, expression }
+  })
+}
+
+function emitMatch(subject, arms) {
+  const defaultIndex = arms.findIndex(arm => arm.pattern === '_')
+
+  if (defaultIndex === -1) {
+    throw new SyntaxError('match requires a default `_` arm')
+  }
+
+  if (defaultIndex !== arms.length - 1) {
+    throw new SyntaxError('the default `_` arm must be the last arm in match')
+  }
+
+  const lines = arms.map(arm =>
+    arm.pattern === '_'
+      ? `  return ${arm.expression};`
+      : `  if (__match === (${arm.pattern})) return ${arm.expression};`
+  )
+
+  return `(() => {\n  const __match = ${subject};\n${lines.join('\n')}\n})()`
+}
+
+export function transformMatchExpressions(source) {
+  let result = source
+
+  for (let iteration = 0; iteration < 1000; iteration++) {
+    const tokens = significantTokens(tokenize(result))
+
+    let matchIndex = -1
+    let block = null
+
+    for (let i = tokens.length - 1; i >= 0; i--) {
+      if (!isMatchKeyword(tokens, i)) continue
+
+      const candidate = findMatchBlock(result, tokens, i)
+      if (candidate) {
+        matchIndex = i
+        block = candidate
+        break
+      }
+    }
+
+    if (matchIndex === -1) return result
+
+    const subject = text(result, block.subjectTokens).trim()
+    const bodyTokens = tokens.slice(block.braceOpen + 1, block.braceClose)
+    const arms = parseArms(result, bodyTokens)
+    const replacement = emitMatch(subject, arms)
+
+    const start = tokens[matchIndex].start
+    const end = tokens[block.braceClose].end
+
+    result = result.slice(0, start) + replacement + result.slice(end)
+  }
+
+  throw new SyntaxError('match transformation exceeded 1000 iterations')
+}