@algosail/parser 0.0.1

package/index.js

@@ -0,0 +1,30 @@
+import { getSailTokenizer } from './lib/sailTokenizer.js'
+import { getJsTokenizer } from './lib/jsTokenizer.js'
+import { buildSymbolTable } from './lib/symbolTable.js'
+import { parseJsDoc } from './lib/jsDoc.js'
+
+export async function createParser() {
+  const sailTokenizer = await getSailTokenizer()
+  const jsTokenizer = await getJsTokenizer()
+  const cache = new Map()
+
+  const parser = {
+    parseSail,
+    parseJs,
+    sailTokenizer,
+    jsTokenizer,
+    cache,
+  }
+
+  async function parseSail(uri, text) {
+    const res = await buildSymbolTable(uri, text, parser)
+    cache.set(uri, res)
+    return res
+  }
+
+  async function parseJs(uri, text) {
+    const res = await parseJsDoc(uri, text, parser)
+    cache.set(uri, res)
+    return res
+  }
+}

package/lib/group.js

@@ -0,0 +1,58 @@
+import { Query } from 'web-tree-sitter'
+
+import { capture, field, childrenOfType, firstOfType } from './utils.js'
+import { groupDef, tagDef, comment } from './tokens.js'
+
+export const getTagNodes = (rootNode, language) => {
+  const query = new Query(language, `(group) @group`)
+  const matches = query.matches(rootNode)
+  const nodes = matches
+    .map((match) => capture(match.captures, 'group'))
+    .filter(Boolean)
+    .map(parseGroup)
+
+  return getTable(nodes)
+}
+
+function parseGroup(node) {
+  return {
+    name: groupDef(field(node, 'def')),
+    doc: comment(field(node, 'doc')),
+    params: childrenOfType(node, 'type_var').map((n) => n.text),
+    tags: childrenOfType(node, 'tag').map(parseTag),
+    startPosition: node.startPosition,
+    endPosition: node.endPosition,
+  }
+}
+
+function parseTag(node) {
+  return {
+    name: tagDef(field(node, 'def')),
+    doc: comment(field(node, 'doc')),
+    param: firstOfType(node, 'type_var')?.text ?? null,
+    startPosition: node.startPosition,
+    endPosition: node.endPosition,
+  }
+}
+
+function getTable(nodes) {
+  const groups = {}
+  const tags = {}
+
+  for (const group of nodes) {
+    groups[group.name] = group
+
+    for (const tag of group.tags) {
+      tags[tag.name] = {
+        ...tag,
+        group: {
+          name: group.name,
+          startPosition: group.startPosition,
+          endPosition: group.endPosition,
+        },
+      }
+    }
+  }
+
+  return { groups, tags }
+}

package/lib/imports.js

@@ -0,0 +1,30 @@
+import { Query } from 'web-tree-sitter'
+import { capture, field } from './utils.js'
+import { pathDef, moduleDef } from './tokens.js'
+
+export function getImportNodes(rootNode, language) {
+  const query = new Query(language, `(import) @import`)
+  const matches = query.matches(rootNode)
+  return matches
+    .map((match) => capture(match.captures, 'import'))
+    .filter(Boolean)
+    .map(parseImport)
+}
+
+export function parseImport(node) {
+  const path = pathDef(field(node, 'path'))
+  return {
+    type: getImportType(path),
+    path,
+    module: moduleDef(field(node, 'module')),
+    startPosition: node.startPosition,
+    endPosition: node.endPosition,
+  }
+}
+
+function getImportType(path) {
+  if (path.startsWith('./') || path.startsWith('../') || path.startsWith('/')) {
+    return 'file'
+  }
+  return 'package'
+}

package/lib/jsDoc.js

@@ -0,0 +1,120 @@
+import { Query } from 'web-tree-sitter'
+import { capture } from './utils.js'
+import { getTagNodes } from './group.js'
+import { getMapNodes } from './map.js'
+import { getWordNodes } from './word.js'
+
+export async function parseJsDoc(uri, text, parser) {
+  const { tokenizer: jsTokenizer, language: jsLanguage } = parser.jsTokenizer
+  const { tokenizer: sailTokenizer, language: sailLanguage } = parser.sailTokenizer
+
+  const jsTree = jsTokenizer.parse(text)
+  const query = new Query(jsLanguage, '(comment) @comment')
+  const matches = query.matches(jsTree.rootNode)
+
+  const result = { modules: {}, groups: {}, tags: {}, maps: {}, words: [] }
+
+  for (const match of matches) {
+    const commentNode = capture(match.captures, 'comment')
+    if (!commentNode) continue
+
+    const extracted = extractSailBlock(commentNode)
+    if (!extracted) continue
+
+    const { sailText, rowOffset, colOffset } = extracted
+
+    const sailTree = sailTokenizer.parse(sailText)
+    const sailRoot = sailTree.rootNode
+
+    const { groups, tags } = getTagNodes(sailRoot, sailLanguage)
+    const { maps } = getMapNodes(sailRoot, sailLanguage)
+    const { words } = getWordNodes(sailRoot, sailLanguage)
+
+    for (const [name, group] of Object.entries(groups)) {
+      result.groups[name] = {
+        ...group,
+        startPosition: offsetPosition(group.startPosition, rowOffset, colOffset),
+        endPosition: offsetPosition(group.endPosition, rowOffset, colOffset),
+        tags: group.tags.map((tag) => ({
+          ...tag,
+          startPosition: offsetPosition(tag.startPosition, rowOffset, colOffset),
+          endPosition: offsetPosition(tag.endPosition, rowOffset, colOffset),
+        })),
+      }
+    }
+
+    for (const [name, tag] of Object.entries(tags)) {
+      result.tags[name] = {
+        ...tag,
+        startPosition: offsetPosition(tag.startPosition, rowOffset, colOffset),
+        endPosition: offsetPosition(tag.endPosition, rowOffset, colOffset),
+        group: {
+          ...tag.group,
+          startPosition: offsetPosition(tag.group.startPosition, rowOffset, colOffset),
+          endPosition: offsetPosition(tag.group.endPosition, rowOffset, colOffset),
+        },
+      }
+    }
+
+    for (const [name, map] of Object.entries(maps)) {
+      result.maps[name] = {
+        ...map,
+        startPosition: offsetPosition(map.startPosition, rowOffset, colOffset),
+        endPosition: offsetPosition(map.endPosition, rowOffset, colOffset),
+        fields: map.fields.map((fieldItem) => ({
+          ...fieldItem,
+          startPosition: offsetPosition(fieldItem.startPosition, rowOffset, colOffset),
+          endPosition: offsetPosition(fieldItem.endPosition, rowOffset, colOffset),
+        })),
+      }
+    }
+
+    result.words.push(...words)
+  }
+
+  return result
+}
+
+function offsetPosition(position, rowOffset, colOffset) {
+  return {
+    row: position.row + rowOffset,
+    column: position.row === 0 ? position.column + colOffset : position.column,
+  }
+}
+
+function extractSailBlock(commentNode) {
+  const text = commentNode.text
+
+  if (!text.startsWith('/**')) return null
+  if (!text.includes('@sail')) return null
+
+  const sailIndex = text.indexOf('@sail')
+
+  const beforeSail = text.slice(0, sailIndex)
+  const sailLineIndex = (beforeSail.match(/\n/g) ?? []).length
+
+  const afterSail = text.slice(sailIndex + '@sail'.length)
+  const lastClose = afterSail.lastIndexOf('*/')
+  const rawContent = lastClose !== -1 ? afterSail.slice(0, lastClose) : afterSail
+
+  const lines = rawContent.split('\n').map((line) => line.replace(/^\s*\*\s?/, ''))
+
+  let firstNonEmpty = 0
+  while (firstNonEmpty < lines.length && lines[firstNonEmpty].trim() === '') {
+    firstNonEmpty++
+  }
+
+  let lastNonEmpty = lines.length - 1
+  while (lastNonEmpty >= firstNonEmpty && lines[lastNonEmpty].trim() === '') {
+    lastNonEmpty--
+  }
+
+  const sailLines = lines.slice(firstNonEmpty, lastNonEmpty + 1)
+  if (sailLines.length === 0) return null
+
+  const rowOffset = commentNode.startPosition.row + sailLineIndex + firstNonEmpty + 1
+  const colOffset = commentNode.startPosition.column + 3
+  const sailText = sailLines.join('\n')
+
+  return { sailText, rowOffset, colOffset }
+}

package/lib/jsTokenizer.js

@@ -0,0 +1,11 @@
+import { Parser, Language } from 'web-tree-sitter'
+
+export async function getJsTokenizer() {
+  await Parser.init()
+  const tokenizer = new Parser()
+  const language = await Language.load(
+    'node_modules/tree-sitter-javascript/tree-sitter-javascript.wasm',
+  )
+  tokenizer.setLanguage(language)
+  return { tokenizer, language }
+}

package/lib/load.js

@@ -0,0 +1,110 @@
+import { readFile, access } from 'node:fs/promises'
+import { createRequire } from 'node:module'
+import { dirname, extname, join, resolve } from 'node:path'
+
+function parsePackageImport(packageImport) {
+  if (packageImport.startsWith('@')) {
+    const parts = packageImport.split('/')
+    const name = `${parts[0]}/${parts[1]}`
+    const path = parts.length > 2 ? `./${parts.slice(2).join('/')}` : `./`
+    return { name, path }
+  }
+
+  const slashIdx = packageImport.indexOf('/')
+  if (slashIdx === -1) return { name: packageImport, path: '.' }
+
+  return {
+    name: packageImport.slice(0, slashIdx),
+    path: `./${packageImport.slice(slashIdx + 1)}`,
+  }
+}
+
+function findPackageDir(execPath, packageImport) {
+  const { name } = parsePackageImport(packageImport)
+  const require = createRequire(execPath)
+  const packageJsonPath = require.resolve(`${name}/package.json`)
+  return dirname(packageJsonPath)
+}
+
+function resolveConditionEntry(entry) {
+  if (typeof entry === 'string') return entry
+  if (!entry || typeof entry !== 'object') return null
+
+  if (Object.prototype.hasOwnProperty.call(entry, 'sail')) {
+    const value = entry.sail
+    if (typeof value === 'object' && value !== null) {
+      const resolved = resolveConditionEntry(value)
+      if (resolved !== null) return resolved
+    } else if (typeof value === 'string') {
+      return value
+    }
+  }
+
+  return null
+}
+
+function resolveExportsCondition(exports, subpath) {
+  if (!exports) return null
+
+  if (typeof exports === 'string') {
+    return subpath === '.' ? exports : null
+  }
+
+  const keys = Object.keys(exports)
+  const isSubpathMap = keys.length > 0 && keys[0].startsWith('.')
+
+  let entry
+  if (isSubpathMap) {
+    entry = exports[subpath]
+    if (entry == null) return null
+  } else {
+    entry = exports
+  }
+
+  return resolveConditionEntry(entry)
+}
+
+export async function resolveModulePath(execPath, packageImport) {
+  const { name, path } = parsePackageImport(packageImport)
+
+  const packageDir = findPackageDir(execPath, packageImport)
+  const packageJson = JSON.parse(await readFile(join(packageDir, 'package.json'), 'utf8'))
+
+  const sailPath = resolveExportsCondition(packageJson.exports, path)
+  if (sailPath) {
+    return readFile(join(packageDir, sailPath), 'utf8')
+  }
+
+  const importId = path === '.' ? name : `${name}/${path.slice(2)}`
+  const absolutePath = createRequire(execPath).resolve(importId)
+  return { type: 'js', path: absolutePath }
+}
+
+export async function resolveFilePath(execPath, filePath) {
+  const base = dirname(execPath)
+  const ext = extname(filePath)
+
+  if (ext === '.sail') {
+    const absolutePath = resolve(base, filePath)
+    return { type: 'sail', path: absolutePath }
+  }
+
+  if (ext === '.js') {
+    const absolutePath = resolve(base, filePath)
+    return { type: 'js', path: absolutePath }
+  }
+
+  const sailPath = resolve(base, `${filePath}.sail`)
+  try {
+    await access(sailPath)
+    return { type: 'sail', path: sailPath }
+  } catch {}
+
+  const jsPath = resolve(base, `${filePath}.js`)
+  try {
+    await access(jsPath)
+    return { type: 'js', path: jsPath }
+  } catch {}
+
+  throw new Error(`Cannot find file "${filePath}" from "${base}"`)
+}

package/lib/map.js

@@ -0,0 +1,52 @@
+import { Query } from 'web-tree-sitter'
+import { capture, fieldText, field, childrenOfType } from './utils.js'
+import { mapDef, fieldDef, comment } from './tokens.js'
+import { litType, groupType, mapType, moduleMapType } from './types.js'
+
+export function getMapNodes(rootNode, language) {
+  const query = new Query(language, `(map) @map`)
+  const matches = query.matches(rootNode)
+  const nodes = matches
+    .map((match) => capture(match.captures, 'map'))
+    .filter(Boolean)
+    .map(parseMap)
+
+  const maps = {}
+  for (const node of nodes) {
+    maps[node.name] = node
+  }
+  return { maps }
+}
+
+function parseMap(node) {
+  return {
+    name: mapDef(field(node, 'def')),
+    doc: comment(field(node, 'doc')),
+    fields: childrenOfType(node, 'field').map(parseMapField),
+    startPosition: node.startPosition,
+    endPosition: node.endPosition,
+  }
+}
+
+function parseMapField(node) {
+  return {
+    name: fieldDef(field(node, 'key')),
+    type: patseFieldType(field(node, 'type')),
+    doc: comment(field(node, 'doc')),
+    startPosition: node.startPosition,
+    endPosition: node.endPosition,
+  }
+}
+
+function patseFieldType(node) {
+  switch (node.type) {
+    case 'group_type':
+      return groupType(node)
+    case 'map_ref':
+      return mapType(node)
+    case 'module_map_ref':
+      return moduleMapType(node)
+    default:
+      return litType(node)
+  }
+}

package/lib/sailTokenizer.js

@@ -0,0 +1,9 @@
+import { Parser, Language } from 'web-tree-sitter'
+
+export async function getSailTokenizer() {
+  await Parser.init()
+  const tokenizer = new Parser()
+  const language = await Language.load('node_modules/@algosail/tree-sitter/tree-sitter-sail.wasm')
+  tokenizer.setLanguage(language)
+  return { tokenizer, language }
+}

package/lib/symbolTable.js

@@ -0,0 +1,95 @@
+import { readFile } from 'node:fs/promises'
+
+import { getImportNodes } from './imports.js'
+import { getTagNodes } from './group.js'
+import { getMapNodes } from './map.js'
+import { getWordNodes } from './word.js'
+import { resolveModulePath, resolveFilePath } from './load.js'
+
+export async function buildSymbolTable(uri, text, parser) {
+  const rootNode = parser.parseSail.tokenizer.parse(text)
+  const imports = getImportNodes(rootNode, parser.parseSail.language)
+  const { groups, tags } = getTagNodes(rootNode, parser.parseSail.language)
+  const { maps } = getMapNodes(rootNode, parser.parseSail.language)
+  const { words } = getWordNodes(rootNode, parser.parseSail.language)
+  const errors = getErrors(rootNode)
+
+  const modules = await getModules(imports, uri, parser)
+
+  return { modules, groups, tags, maps, words, errors }
+}
+
+function getErrors(rootNode) {
+  const errors = []
+  const stack = [...rootNode.children]
+
+  while (stack.length > 0) {
+    const node = stack.pop()
+
+    if (node.type === 'ERROR') {
+      errors.push({
+        type: 'error',
+        text: node.text,
+        startPosition: node.startPosition,
+        endPosition: node.endPosition,
+      })
+    } else if (node.isMissing) {
+      errors.push({
+        type: 'missing',
+        text: node.type,
+        startPosition: node.startPosition,
+        endPosition: node.endPosition,
+      })
+    }
+
+    if (node.hasError) {
+      stack.push(...node.children)
+    }
+  }
+
+  return errors
+}
+
+async function getModules(imports, uri, parser) {
+  const modules = {}
+
+  for (const imp of imports) {
+    if (imp.type === 'package') {
+      const moduleUri = await resolveModulePath(imp.path, uri)
+      const content = await readFile(moduleUri, 'utf8')
+      const res = await parser.parseSail(moduleUri, content)
+      modules[imp.module] = {
+        ...res,
+        uri: moduleUri,
+        startPosition: imp.startPosition,
+        endPosition: imp.endPosition,
+      }
+    }
+
+    if (imp.type === 'file') {
+      const { type, path: fileUri } = await resolveFilePath(imp.path, execPath)
+      const content = await readFile(fileUri, 'utf8')
+      if (type === 'sail') {
+        const res = await parser.parseSail(fileUri, content)
+        modules[imp.module] = {
+          ...res,
+          uri: fileUri,
+          startPosition: imp.startPosition,
+          endPosition: imp.endPosition,
+        }
+      }
+
+      if (type === 'js') {
+        const res = await parser.parseJs(fileUri, content)
+        modules[imp.module] = {
+          ...res,
+          uri: fileUri,
+          startPosition: imp.startPosition,
+          endPosition: imp.endPosition,
+        }
+      }
+    }
+  }
+
+  return modules
+}

package/lib/tokens.js

@@ -0,0 +1,79 @@
+import { firstOfType, childrenOfField } from './utils.js'
+export const pathDef = (node) => node?.text?.slice(1) ?? null
+
+export const moduleDef = (node) => node?.text?.slice(1) ?? null
+export const moduleRef = (node) => node?.text?.slice(1) ?? null
+export const groupDef = (node) => node?.text?.slice(1) ?? null
+export const groupRef = (node) => node?.text?.slice(1) ?? null
+export const tagDef = (node) => node?.text?.slice(1) ?? null
+export const tagRef = (node) => node?.text?.slice(1) ?? null
+export const tagPattern = (node) => node?.text?.slice(1) ?? null
+export const mapDef = (node) => node?.text?.slice(1) ?? null
+export const mapRef = (node) => node?.text?.slice(1) ?? null
+export const fieldDef = (node) => node?.text?.slice(1) ?? null
+
+export const fieldRef = (node) => {
+  if (!node) return null
+  const [map, field] = node.text.split('.')
+  return { map: mapRef(map), field }
+}
+
+export const wordDef = (node) => node?.text?.slice(1) ?? null
+export const wordRef = (node) => node?.text?.slice(1) ?? null
+
+export const moduleGroupRef = (node) => {
+  if (!node) return null
+  const [module, group] = node.text.split('&')
+  return { module: module.slice(1), group }
+}
+
+export const moduleTagRef = (node) => {
+  if (!node) return null
+  const [module, tag] = node.text.split('#')
+  return { module: module.slice(1), tag }
+}
+
+export const moduleTagPattern = (node) => {
+  if (!node) return null
+  const [module, tag] = node.text.split('_')
+  return { module: module.slice(1), tag }
+}
+
+export const moduleMapRef = (node) => {
+  if (!node) return null
+  const [module, map] = node.text.split('$')
+  return { module: module.slice(1), map }
+}
+
+export const moduleFieldRef = (node) => {
+  if (!node) return null
+  const [module, fieldRef] = node.text.split('$')
+  const [map, field] = fieldRef.split('.')
+  return { module: module.slice(1), map, field }
+}
+
+export const moduleWordRef = (node) => {
+  if (!node) return null
+  const [module, word] = node.text.split('/')
+  return { module: module.slice(1), word }
+}
+
+export const type = (node) => node?.text ?? null
+export const typeVar = (node) => node?.text ?? null
+export const spread = (node) => node?.text.slice(2) ?? null
+
+export const effectAdd = (node) => node?.text.slice(1) ?? null
+export const effectRemove = (node) => node?.text.slice(1) ?? null
+
+export const slotWrite = (node) => node?.text.slice(1) ?? null
+export const slotRead = (node) => node?.text.slice(1) ?? null
+
+export const rawString = (node) => node?.text.slice(1, -1) ?? null
+export const rawValue = (node) => node?.text ?? null
+
+export const comment = (commentNode) => {
+  if (!commentNode) return null
+  const content = firstOfType(commentNode, 'comment_content')
+  if (!content) return null
+  return content.text.trim()
+}