agent-simple-english 0.1.0

package/src/engine/sentences.ts

@@ -0,0 +1,285 @@
+export interface Sentence {
+  readonly text: string
+  readonly line: number
+  readonly column: number
+  readonly endLine: number
+  readonly startOffset: number
+  readonly endOffset: number
+  readonly contentRanges: readonly {
+    readonly start: number
+    readonly end: number
+  }[]
+}
+
+const CLOSING_DELIMITERS = new Set([
+  '"',
+  "'",
+  "’",
+  "”",
+  "»",
+  "›",
+  ")",
+  "]",
+  "}",
+  "*",
+  "_",
+  "~",
+  "`",
+])
+
+function valueAt(values: Int32Array | Uint32Array, index: number): number {
+  const value = values[index]
+  if (value === undefined) throw new RangeError(`Array index ${index} is out of bounds`)
+  return value
+}
+
+function sentinelAt(values: Int32Array, index: number): number {
+  return values[index] ?? -1
+}
+
+function markdownDelimiterEnds(text: string): {
+  readonly parentheses: Int32Array
+  readonly brackets: Int32Array
+  readonly linkSuffixes: Int32Array
+} {
+  const parentheses = new Int32Array(text.length).fill(-1)
+  const brackets = new Int32Array(text.length).fill(-1)
+  const linkSuffixes = new Int32Array(text.length).fill(-1)
+  const parenthesisStack: number[] = []
+  const bracketStack: number[] = []
+  let opaqueDelimiter: string | undefined
+
+  for (let index = 0; index < text.length; index += 1) {
+    const character = text[index]
+    if (opaqueDelimiter !== undefined) {
+      if (character === "\\") {
+        index += 1
+      } else if (character === opaqueDelimiter) {
+        opaqueDelimiter = undefined
+      }
+      continue
+    }
+
+    switch (character) {
+      case "\\":
+        index += 1
+        break
+      case '"':
+      case "'": {
+        const opening = parenthesisStack[parenthesisStack.length - 1]
+        const isLinkTitle =
+          opening !== undefined && text[opening - 1] === "]" && /\s/.test(text[index - 1] ?? "")
+        if (isLinkTitle) opaqueDelimiter = character
+        break
+      }
+      case "<": {
+        const opening = parenthesisStack[parenthesisStack.length - 1]
+        const isAngleDestination =
+          opening !== undefined && text[opening - 1] === "]" && index === opening + 1
+        if (isAngleDestination) opaqueDelimiter = ">"
+        break
+      }
+      case "(":
+        parenthesisStack.push(index)
+        break
+      case ")": {
+        const opening = parenthesisStack.pop()
+        if (opening !== undefined) parentheses[opening] = index + 1
+        break
+      }
+      case "[":
+        bracketStack.push(index)
+        break
+      case "]": {
+        const opening = bracketStack.pop()
+        if (opening !== undefined) brackets[opening] = index + 1
+        break
+      }
+    }
+  }
+
+  for (let index = 0; index < text.length - 1; index += 1) {
+    if (text[index] !== "]") continue
+    const suffixStart = index + 1
+    const parenthesisEnd = valueAt(parentheses, suffixStart)
+    const bracketEnd = valueAt(brackets, suffixStart)
+    if (text[suffixStart] === "(" && parenthesisEnd >= 0) {
+      linkSuffixes[suffixStart] = parenthesisEnd
+    } else if (text[suffixStart] === "[" && bracketEnd >= 0) {
+      linkSuffixes[suffixStart] = bracketEnd
+    }
+  }
+
+  return { parentheses, brackets, linkSuffixes }
+}
+
+function sentenceTerminatorEnds(text: string): number[] {
+  const { parentheses, brackets, linkSuffixes } = markdownDelimiterEnds(text)
+  const closingRuns = new Uint32Array(text.length + 1)
+  const closingBracketCounts = new Uint32Array(text.length + 1)
+  const referenceRuns = new Int32Array(text.length).fill(-1)
+  const ends: number[] = []
+  closingRuns[text.length] = text.length
+
+  for (let index = text.length - 1; index >= 0; index -= 1) {
+    closingRuns[index] = CLOSING_DELIMITERS.has(text[index] ?? "")
+      ? valueAt(closingRuns, index + 1)
+      : index
+    closingBracketCounts[index] =
+      valueAt(closingBracketCounts, index + 1) + (text[index] === "]" ? 1 : 0)
+  }
+
+  for (let index = text.length - 1; index >= 0; index -= 1) {
+    const bracketEnd = valueAt(brackets, index)
+    if (text[index] !== "[" || bracketEnd < 0) continue
+    let end = valueAt(closingRuns, bracketEnd)
+    const referenceEnd = sentinelAt(referenceRuns, end)
+    if (text[end] === "[" && referenceEnd >= 0) end = referenceEnd
+    referenceRuns[index] = end
+  }
+
+  for (let index = 0; index < text.length; index += 1) {
+    const linkSuffixEnd = valueAt(linkSuffixes, index)
+    if (linkSuffixEnd >= 0) {
+      index = linkSuffixEnd - 1
+      continue
+    }
+    if (text[index] !== "." && text[index] !== "!" && text[index] !== "?") continue
+
+    let punctuationEnd = index + 1
+    while (
+      text[punctuationEnd] === "." ||
+      text[punctuationEnd] === "!" ||
+      text[punctuationEnd] === "?"
+    ) {
+      punctuationEnd += 1
+    }
+
+    let end = valueAt(closingRuns, punctuationEnd)
+    const closedLinkLabel =
+      valueAt(closingBracketCounts, punctuationEnd) > valueAt(closingBracketCounts, end)
+
+    if (closedLinkLabel && text[end] === "(") {
+      const parenthesisEnd = valueAt(parentheses, end)
+      if (parenthesisEnd < 0) {
+        index = punctuationEnd - 1
+        continue
+      }
+      end = valueAt(closingRuns, parenthesisEnd)
+    }
+
+    const referenceEnd = sentinelAt(referenceRuns, end)
+    if (text[end] === "[" && referenceEnd >= 0) end = referenceEnd
+    if (end === text.length || /\s/.test(text[end] ?? "")) {
+      ends.push(end)
+      index = end - 1
+    } else {
+      index = punctuationEnd - 1
+    }
+  }
+
+  return ends
+}
+
+// A sentence starts at the first non-whitespace character and ends at
+// terminal punctuation and closing delimiters, at a blank line, or at EOF.
+// Sentences may span lines; position is where the sentence starts (1-based).
+export function segmentSentences(
+  lines: readonly string[],
+  sourceText: string = lines.join("\n"),
+  structuralBlanks: readonly boolean[] = lines.map((line) => line.trim() === ""),
+): Sentence[] {
+  const sentences: Sentence[] = []
+  const lineOffsets = [0]
+  for (let index = 0; index < sourceText.length; index++) {
+    if (sourceText[index] === "\n") lineOffsets.push(index + 1)
+  }
+  let open: {
+    line: number
+    column: number
+    endLine: number
+    startOffset: number
+    endOffset: number
+    parts: string[]
+    contentRanges: Array<{ start: number; end: number }>
+  } | null = null
+
+  const appendPart = (part: string, startOffset: number) => {
+    if (!open) return
+    const leadingWhitespace = part.length - part.trimStart().length
+    const text = part.trim()
+    open.parts.push(text)
+    if (text === "") return
+    const start = startOffset + leadingWhitespace
+    const end = start + text.length
+    open.contentRanges.push({ start, end })
+    open.endOffset = end
+  }
+
+  const close = () => {
+    if (!open) return
+    const text = open.parts.join(" ").trim()
+    if (text !== "") {
+      sentences.push({
+        text,
+        line: open.line,
+        column: open.column,
+        endLine: open.endLine,
+        startOffset: open.startOffset,
+        endOffset: open.endOffset,
+        contentRanges: open.contentRanges,
+      })
+    }
+    open = null
+  }
+
+  lines.forEach((raw, index) => {
+    if (raw.trim() === "") {
+      if (structuralBlanks[index] ?? true) close()
+      return
+    }
+    let offset = 0
+    for (const end of sentenceTerminatorEnds(raw)) {
+      const part = raw.slice(offset, end)
+      if (!open) {
+        const indent = part.length - part.trimStart().length
+        const startOffset = (lineOffsets[index] ?? 0) + offset + indent
+        open = {
+          line: index + 1,
+          column: offset + indent + 1,
+          endLine: index + 1,
+          startOffset,
+          endOffset: startOffset,
+          parts: [],
+          contentRanges: [],
+        }
+      }
+      open.endLine = index + 1
+      appendPart(part, (lineOffsets[index] ?? 0) + offset)
+      close()
+      offset = end
+    }
+
+    const rest = raw.slice(offset)
+    if (rest.trim() !== "") {
+      if (!open) {
+        const indent = rest.length - rest.trimStart().length
+        const startOffset = (lineOffsets[index] ?? 0) + offset + indent
+        open = {
+          line: index + 1,
+          column: offset + indent + 1,
+          endLine: index + 1,
+          startOffset,
+          endOffset: startOffset,
+          parts: [],
+          contentRanges: [],
+        }
+      }
+      open.endLine = index + 1
+      appendPart(rest, (lineOffsets[index] ?? 0) + offset)
+    }
+  })
+  close()
+
+  return sentences
+}

package/src/engine/tagger.ts

@@ -0,0 +1,8 @@
+export interface TaggedToken {
+  readonly text: string
+  readonly pos: string
+  readonly lemma: string
+  readonly offset: number
+}
+
+export type Tagger = (text: string) => readonly TaggedToken[]

package/src/engine/tokens.ts

@@ -0,0 +1,2 @@
+export const TOKEN_CHARACTER_PATTERN = "[A-Za-z0-9_'’-]"
+export const TOKEN_RUN_PATTERN = new RegExp(`${TOKEN_CHARACTER_PATTERN}+`, "g")

package/src/engine/types.ts

@@ -0,0 +1,45 @@
+import type { Dictionary } from "../dictionary/schema.ts"
+import type { RuleId } from "./rules/registry.ts"
+import type { Tagger } from "./tagger.ts"
+
+export type LintKind = "prose-file" | "slash-source" | "hash-source" | "commit-message"
+
+export type Severity = "hard" | "soft"
+
+export type RuleSetting = Severity | "off"
+
+export interface Violation {
+  readonly ruleId: RuleId
+  readonly severity: Severity
+  readonly message: string
+  readonly suggestions?: readonly string[]
+  readonly line: number
+  readonly column: number
+  readonly suggestion?: string
+}
+
+export type SourceDialect = "general" | "shell"
+
+export interface LintOptions {
+  readonly rules?: Partial<Record<RuleId, RuleSetting>>
+  readonly maxSentenceWords?: number
+  readonly dictionary?: Dictionary
+  // POS tagger for the verb-form rules and POS-aware dictionary entries.
+  readonly tagger?: Tagger
+  readonly sourceDialect?: SourceDialect
+  /**
+   * The prior document text used to report only new violations.
+   * The engine compares sentence-scoped and paragraph-scoped violations structurally.
+   * Omit this value to lint the current document in full.
+   * Violation positions refer to the current text.
+   */
+  readonly previousText?: string
+}
+
+export interface LintReport {
+  readonly violations: readonly Violation[]
+  readonly summary: {
+    readonly total: number
+    readonly hard: number
+  }
+}