@panagos/jojoscript 0.10.1 → 1.0.0

package/src/simulate.js

@@ -0,0 +1,156 @@
+import { explainPipelines } from './pipeline-explain.js'
+
+const DEFAULT_INPUT = 1000
+const EFFECT_VOLUME_WARNING = 1000
+const MEMORY_WARNING = 10000
+
+const NARROWING_STAGES = new Set(['filter', 'filterAsync', 'unique', 'distinctBy'])
+const SINGLE_RESULT_STAGES = new Set(['count', 'first', 'find', 'some', 'every', 'reduce'])
+const MATERIALIZING_STAGES = new Set(['sort', 'sortBy', 'groupBy', 'partition'])
+
+function integerArgument(stage) {
+  return /^\d+$/.test(stage.args[0] ?? '') ? Number(stage.args[0]) : null
+}
+
+/*
+ * Upper bound on the items a stage emits, given the items it receives.
+ * `null` means the count cannot be known statically.
+ */
+function itemsOut(stage, items) {
+  if (items === null) return SINGLE_RESULT_STAGES.has(stage.type) ? 1 : null
+  const size = integerArgument(stage)
+
+  switch (stage.type) {
+    case 'take': return size === null ? null : Math.min(size, items)
+    case 'skip': return size === null ? null : Math.max(items - size, 0)
+    case 'chunk':
+    case 'batch': return size === null ? null : Math.ceil(items / size)
+    case 'window': return size === null ? null : Math.max(items - size + 1, 0)
+    case 'flatMap':
+    case 'groupBy': return null
+    case 'partition': return 2
+    default: return SINGLE_RESULT_STAGES.has(stage.type) ? 1 : items
+  }
+}
+
+function exactOut(stage, exact) {
+  return exact && !NARROWING_STAGES.has(stage.type)
+}
+
+function format(items, exact) {
+  if (items === null) return 'unknown'
+  return `${exact ? '' : '≤'}${items.toLocaleString('en-US')}`
+}
+
+function simulateStages(explanation, input) {
+  let items = input
+  let exact = true
+
+  return explanation.stages.slice(1).map(stage => {
+    const effects = [...stage.effects, ...stage.effectfulFunctions]
+    const simulated = {
+      index: stage.index,
+      id: stage.id,
+      type: stage.type,
+      items,
+      exact,
+      workers: stage.type === 'parallel' ? integerArgument(stage) : null,
+      effectCalls: Object.fromEntries(effects.map(name => [name, items])),
+      checkpoint: stage.type === 'checkpoint' ? stage.detail : null
+    }
+
+    exact = exactOut(stage, exact)
+    items = itemsOut(stage, items)
+    return simulated
+  })
+}
+
+function findProblems(stages) {
+  const problems = []
+  const checkpoints = stages.filter(stage => stage.checkpoint)
+
+  for (const stage of stages) {
+    if (MATERIALIZING_STAGES.has(stage.type) && stage.items >= MEMORY_WARNING) {
+      const amount = stage.exact ? `all ${format(stage.items, true)}` : `up to ${format(stage.items, true)}`
+      problems.push({ stage: stage.index, message: `${stage.type} holds ${amount} items in memory before continuing` })
+    }
+
+    for (const [name, calls] of Object.entries(stage.effectCalls)) {
+      if (calls !== null && calls >= EFFECT_VOLUME_WARNING) {
+        problems.push({ stage: stage.index, message: `up to ${format(calls, true)} calls to effect "${name}"` })
+      }
+
+      const nextCheckpoint = checkpoints.find(checkpoint => checkpoint.index > stage.index)
+      if (nextCheckpoint) {
+        problems.push({
+          stage: stage.index,
+          message: `effect "${name}" runs before checkpoint "${nextCheckpoint.checkpoint}": items processed before a failure are fetched again on retry (at-least-once)`
+        })
+      }
+    }
+  }
+
+  if (!checkpoints.length && stages.some(stage => Object.keys(stage.effectCalls).length)) {
+    problems.push({ stage: null, message: 'pipeline has effects but no checkpoint: a failure re-runs every effect from the start' })
+  }
+
+  return problems
+}
+
+function totalEffectCalls(stages) {
+  const totals = {}
+  for (const stage of stages) {
+    for (const [name, calls] of Object.entries(stage.effectCalls)) {
+      totals[name] = name in totals && (totals[name] === null || calls === null) ? null : (totals[name] ?? 0) + calls
+    }
+  }
+  return totals
+}
+
+/*
+ * A static what-if for `input` records: no stage or effect is executed.
+ * Counts are upper bounds once a filtering stage has run.
+ */
+export function simulatePipelines(source, options = {}) {
+  const input = options.input ?? DEFAULT_INPUT
+
+  return explainPipelines(source).map(explanation => {
+    const stages = simulateStages(explanation, input)
+    return {
+      name: explanation.name,
+      input,
+      stages,
+      effectCalls: totalEffectCalls(stages),
+      checkpoints: stages.filter(stage => stage.checkpoint).length,
+      problems: findProblems(stages)
+    }
+  })
+}
+
+function describeOperation(stage) {
+  const parts = [`${format(stage.items, stage.exact)} items in`]
+  if (stage.workers !== null) parts.push(`${stage.workers} workers`)
+  for (const [name, calls] of Object.entries(stage.effectCalls)) parts.push(`${format(calls, stage.exact)} calls to ${name}`)
+  if (stage.checkpoint) parts.push(`saved as "${stage.checkpoint}"`)
+  return parts.join(', ') + (stage.id ? ` @${stage.id}` : '')
+}
+
+export function renderSimulationText(simulation) {
+  const width = Math.max(...simulation.stages.map(stage => stage.type.length))
+  const problems = simulation.problems.length
+    ? simulation.problems.map(problem => `  ⚠ ${problem.stage === null ? '' : `stage ${problem.stage}: `}${problem.message}`)
+    : ['  none']
+
+  return [
+    `SIMULATION ${simulation.name}`,
+    '',
+    'INPUT',
+    `  ${simulation.input.toLocaleString('en-US')} records`,
+    '',
+    'OPERATIONS',
+    ...simulation.stages.map(stage => `  ${stage.index} ${stage.type.padEnd(width)} ${describeOperation(stage)}`),
+    '',
+    'POTENTIAL PROBLEMS',
+    ...problems
+  ].join('\n')
+}

package/src/type-declarations.js

@@ -0,0 +1,83 @@
+import { maskNonCode } from './lexer.js'
+
+const VARIANT = String.raw`[A-Za-z_$][\w$]*(?:\s*\([^()]*\))?`
+const TYPE_DECLARATION = new RegExp(
+  String.raw`^([ \t]*)(export[ \t]+)?type[ \t]+([A-Za-z_$][\w$]*)[ \t]*=\s*(\|?\s*${VARIANT}(?:\s*\|\s*${VARIANT})*)`,
+  'gm'
+)
+
+/*
+ * `type Result = | Ok(value) | Failed(message)` declares a tagged union.
+ * Variant names must start with an uppercase letter so `match` can tell a
+ * constructor pattern such as `Ok(v)` apart from an ordinary call compared by value.
+ */
+export function findTypeDeclarations(source) {
+  const mask = maskNonCode(source)
+  const regex = new RegExp(TYPE_DECLARATION.source, TYPE_DECLARATION.flags)
+  const declarations = []
+  let match
+
+  while ((match = regex.exec(source))) {
+    if (mask.slice(match.index, match.index + match[0].length) !== match[0]) continue
+
+    const [text, indent, exportPart, name, body] = match
+    declarations.push({
+      name,
+      variants: parseVariants(name, body),
+      indent,
+      exported: Boolean(exportPart),
+      start: match.index,
+      end: match.index + text.length
+    })
+  }
+
+  return declarations
+}
+
+function parseVariants(typeName, body) {
+  const variants = body.split('|').map(part => part.trim()).filter(Boolean).map(part => {
+    const [, name, params] = part.match(/^([A-Za-z_$][\w$]*)\s*(?:\(([^()]*)\))?$/)
+    if (!/^[A-Z]/.test(name)) {
+      throw new SyntaxError(`Variant "${name}" of type "${typeName}" must start with an uppercase letter`)
+    }
+
+    const fields = params?.trim() ? params.split(',').map(field => field.trim()) : []
+    const duplicateField = fields.find((field, index) => fields.indexOf(field) !== index)
+    if (duplicateField) {
+      throw new SyntaxError(`Variant "${name}" declares field "${duplicateField}" more than once`)
+    }
+
+    return { name, fields }
+  })
+
+  const names = variants.map(variant => variant.name)
+  const duplicate = names.find((name, index) => names.indexOf(name) !== index)
+  if (duplicate) {
+    throw new SyntaxError(`Type "${typeName}" declares variant "${duplicate}" more than once`)
+  }
+
+  return variants
+}
+
+export function transformTypeDeclarations(source) {
+  let result = source
+
+  for (const declaration of findTypeDeclarations(source).reverse()) {
+    const exportPart = declaration.exported ? 'export ' : ''
+    const variants = declaration.variants
+      .map(variant => `${variant.name}: ${JSON.stringify(variant.fields).replaceAll(',', ', ')}`)
+      .join(', ')
+    const names = declaration.variants.map(variant => variant.name).join(', ')
+
+    // Keeping the declaration's line count keeps later line numbers stable.
+    const lineBreaks = '\n'.repeat(source.slice(declaration.start, declaration.end).split('\n').length - 1)
+
+    const replacement =
+      `${declaration.indent}${exportPart}const ${declaration.name} = defineType(${JSON.stringify(declaration.name)}, { ${variants} }); ` +
+      `${exportPart}const { ${names} } = ${declaration.name}${lineBreaks}`
+
+    result = result.slice(0, declaration.start) + replacement + result.slice(declaration.end)
+  }
+
+  return result
+}

package/src/why.js

@@ -0,0 +1,146 @@
+import { explainPipelines } from './pipeline-explain.js'
+
+const MATERIALIZING_STAGES = new Set(['sort', 'sortBy', 'groupBy', 'partition'])
+const NARROWING_STAGES = new Set(['filter', 'filterAsync', 'unique', 'distinctBy', 'take', 'skip'])
+
+function label(stage) {
+  return `stage ${stage.index} (${stage.type})`
+}
+
+function capitalize(text) {
+  return text[0].toUpperCase() + text.slice(1)
+}
+
+function isEffectful(stage) {
+  return stage.effects.length > 0 || stage.effectfulFunctions.length > 0
+}
+
+function uses(stage) {
+  return [
+    stage.effects.length ? `effect ${stage.effects.join(', ')}` : null,
+    stage.effectfulFunctions.length ? `effectful function ${stage.effectfulFunctions.join(', ')}` : null
+  ].filter(Boolean).join(' and ')
+}
+
+function determinism(stages) {
+  const effectful = stages.filter(isEffectful)
+  if (!effectful.length) {
+    return {
+      question: 'Why is this pipeline deterministic?',
+      answer: 'No stage uses a declared effect or an effectful function. Plain JavaScript calls inside stages are not analyzed.'
+    }
+  }
+  return {
+    question: 'Why is this pipeline not deterministic?',
+    answer: capitalize(`${effectful.map(stage => `${label(stage)} uses ${uses(stage)}`).join('; ')}.`)
+  }
+}
+
+function laziness(stages) {
+  const blocking = stages.find(stage => MATERIALIZING_STAGES.has(stage.type))
+  if (!blocking) {
+    return {
+      question: 'Why is this pipeline lazy?',
+      answer: 'Every stage passes items on one at a time, so no stage waits for the complete input.'
+    }
+  }
+  return {
+    question: 'Why is this pipeline not lazy?',
+    answer: `${capitalize(label(blocking))} needs the complete input before it can emit anything.`
+  }
+}
+
+function resumability(checkpoints) {
+  if (!checkpoints.length) {
+    return {
+      question: 'Why is this pipeline not resumable?',
+      answer: 'It has no checkpoint stage, so a failure restarts it from the source.'
+    }
+  }
+
+  const sentences = checkpoints.map(checkpoint => {
+    const repeated = checkpoint.index > 2 ? `repeating stages 2-${checkpoint.index - 1}` : 'reading the source again'
+    return `Checkpoint "${checkpoint.detail}" (stage ${checkpoint.index}) saves its items once they all complete; a rerun restores them instead of ${repeated}.`
+  })
+  return { question: 'Why is this pipeline resumable?', answer: sentences.join(' ') }
+}
+
+function ordering(stages) {
+  return stages.filter(isEffectful).flatMap(stage => stages
+    .filter(narrowing => NARROWING_STAGES.has(narrowing.type) && narrowing.index < stage.index)
+    .map(narrowing => ({
+      question: `Why can't stage ${stage.index} be moved before stage ${narrowing.index}?`,
+      answer: `${capitalize(label(stage))} uses ${uses(stage)} and ${label(narrowing)} decides which items reach it: moving it earlier would also run it for items that stage ${narrowing.index} drops.`
+    })))
+}
+
+function parallelism(stages) {
+  return stages
+    .filter(stage => stage.type === 'parallel' && stages.some(next => next.index === stage.index + 1))
+    .map(stage => {
+      const next = stages.find(candidate => candidate.index === stage.index + 1)
+      const concurrency = stage.args[0]
+      return {
+        question: `Why does stage ${stage.index} run in parallel?`,
+        answer: `parallel(${concurrency}) lets up to ${concurrency} items of ${label(next)} be processed at once; results keep their input order.`
+      }
+    })
+}
+
+function replaySafety(stages, checkpoints) {
+  const beforeCheckpoints = checkpoints.flatMap(checkpoint => {
+    const effects = stages
+      .filter(stage => isEffectful(stage) && stage.index < checkpoint.index)
+      .flatMap(stage => [...stage.effects, ...stage.effectfulFunctions].map(name => ({ name, index: stage.index })))
+    if (!effects.length) return []
+
+    const subject = effects.length === 1
+      ? `Effect ${effects[0].name} runs before it (stage ${effects[0].index})`
+      : `Effects ${effects.map(effect => `${effect.name} (stage ${effect.index})`).join(' and ')} run before it`
+    const pronoun = effects.length === 1 ? 'it runs' : 'they run'
+    return [{
+      question: `Why does checkpoint "${checkpoint.detail}" not guarantee replay safety?`,
+      answer: `${subject}: if a run fails before "${checkpoint.detail}" completes, ${pronoun} again on retry (at-least-once, not exactly-once).`
+    }]
+  })
+
+  const last = checkpoints.at(-1)
+  const afterLast = last
+    ? stages.filter(stage => isEffectful(stage) && stage.index > last.index).map(stage => ({
+        question: `Why can stage ${stage.index} repeat after a failure?`,
+        answer: `${capitalize(label(stage))} uses ${uses(stage)} after the last checkpoint "${last.detail}": a rerun restores the checkpoint and runs it again for every restored item.`
+      }))
+    : []
+
+  return [...beforeCheckpoints, ...afterLast]
+}
+
+/*
+ * Plain-language answers derived from the same static analysis as
+ * `jojo explain`, using its stage numbers (the source is stage 1).
+ */
+export function whyPipelines(source) {
+  return explainPipelines(source).map(explanation => {
+    const stages = explanation.stages.slice(1)
+    const checkpoints = stages.filter(stage => stage.type === 'checkpoint')
+
+    return {
+      name: explanation.name,
+      answers: [
+        determinism(stages),
+        laziness(stages),
+        resumability(checkpoints),
+        ...ordering(stages),
+        ...parallelism(stages),
+        ...replaySafety(stages, checkpoints)
+      ]
+    }
+  })
+}
+
+export function renderWhyText(why) {
+  return [
+    `WHY ${why.name}`,
+    ...why.answers.flatMap(({ question, answer }) => ['', question, `→ ${answer}`])
+  ].join('\n')
+}