ja-compromise 0.0.1 → 0.1.0

package/src/02-two/tagset/tags/verbs.js

@@ -0,0 +1,93 @@
+export default {
+  Verb: {
+    not: ['Noun', 'Adjective', 'Adverb', 'Value', 'Expression'],
+  },
+  ConditionalVerb: {
+    is: 'Verb',
+  },
+  Passive: {
+    is: 'Verb',
+  },
+  Causative: {
+    is: 'Verb',
+  },
+  // tense is a facet, not a class: japanese adjectives carry it too
+  // (高かった is a past-tense *adjective*), so these must not imply #Verb
+  PresentTense: {
+    not: ['PastTense'],
+  },
+  Infinitive: {
+    is: 'Verb',
+    not: ['Gerund'],
+  },
+  Imperative: {
+    is: 'Verb',
+  },
+  // the て-form takes its tense from the helper verb that follows it
+  // (読んで いました), so it must not fight with #PastTense
+  Gerund: {},
+  PastTense: {
+    not: ['PresentTense'],
+  },
+  Copula: {
+    is: 'Verb',
+  },
+  Modal: {
+    is: 'Verb',
+    not: ['Infinitive'],
+  },
+  PerfectTense: {
+    is: 'Verb',
+    not: ['Gerund'],
+  },
+  Pluperfect: {
+    is: 'Verb',
+  },
+  Participle: {
+    is: 'PastTense',
+  },
+  PhrasalVerb: {
+    is: 'Verb',
+  },
+  Particle: {
+    is: 'PhrasalVerb',
+    not: ['PastTense', 'PresentTense', 'Copula', 'Gerund'],
+  },
+  Auxiliary: {
+    is: 'Verb',
+    not: ['Conjunction'],
+  },
+  // japanese stacks these onto a single word, so they're facets, not classes
+  Negative: {
+    not: ['Noun', 'Value'],
+  },
+  Polite: {},
+  Volitional: {
+    is: 'Verb',
+  },
+  Potential: {
+    is: 'Verb',
+  },
+  Progressive: {
+    is: 'Verb',
+  },
+  Desire: {
+    is: 'Verb',
+  },
+  Representative: {
+    is: 'Verb',
+  },
+  Presumptive: {},
+  Continuative: {
+    is: 'Verb',
+  },
+  // 連用形 - the 'masu-stem'.  it heads a verb-phrase (泳ぎます) but stands
+  // alone as a noun too - 泳ぎ is 'a swim'
+  VerbStem: {
+    is: 'Noun',
+  },
+  // 勉強, which is a noun until it meets する
+  SuruVerb: {
+    is: 'Noun',
+  },
+}

package/src/_lib.js

@@ -0,0 +1,2 @@
+import nlp from 'compromise/one'
+export default nlp

package/src/_version.js

@@ -0,0 +1 @@
+export default '0.1.0'

package/src/index.js

@@ -0,0 +1,82 @@
+import nlp from './_lib.js'
+import lexicon from './01-one/lexicon/plugin.js'
+import toTerms from './01-one/tokenizer/methods/whitespace.js'
+import romanji from './01-one/romanji/plugin.js'
+import tokenizer from './01-one/tokenizer/plugin.js'
+import output from './01-one/output/plugin.js'
+import tagset from './02-two/tagset/plugin.js'
+import preTagger from './02-two/preTagger/plugin.js'
+import version from './_version.js'
+import { methods as lexMethods } from './01-one/lexicon/api.js'
+import toNumber from './01-one/numbers/kanji-number.js'
+import toKanji from './01-one/numbers/to-kanji.js'
+import numbers from './01-one/numbers/api.js'
+
+nlp.plugin(tokenizer)
+nlp.plugin(tagset)
+nlp.plugin(lexicon)
+nlp.plugin(output)
+nlp.plugin(romanji)
+nlp.plugin(preTagger)
+nlp.plugin(numbers)
+
+const ja = function (txt, lex) {
+  // split sentences
+  let doc = nlp.tokenize(txt, lex)
+  // tokenize terms ourselves - compromise splits on whitespace, which japanese
+  // mostly doesn't use.  every whitespace-chunk still needs splitting, so run
+  // the tokenizer over each of them and keep their surrounding punctuation.
+  doc.document = doc.document.map(a => {
+    let out = []
+    // if the sentence has spaces in it, they're real word-boundaries
+    let spaced = a.length > 1
+    a.forEach(term => {
+      let terms = toTerms(term.text, spaced)
+      if (terms.length === 0) {
+        out.push(term)
+        return
+      }
+      terms[0].pre = (term.pre || '') + terms[0].pre
+      terms[terms.length - 1].post += term.post || ''
+      out = out.concat(terms)
+    })
+    return out
+  })
+  const world = nlp.world()
+  doc.compute(world.hooks)
+  return doc
+}
+
+// copy constructor methods over
+Object.keys(nlp).forEach(k => {
+  if (nlp.hasOwnProperty(k)) {
+    ja[k] = nlp[k]
+  }
+})
+
+// this one is hidden
+Object.defineProperty(ja, '_world', {
+  value: nlp._world,
+  writable: true,
+})
+/** log the decision-making to console */
+ja.verbose = function (set) {
+  let env = typeof process === 'undefined' ? self.env || {} : process.env //use window, in browser
+  env.DEBUG_TAGS = set === 'tagger' || set === true ? true : ''
+  env.DEBUG_MATCH = set === 'match' || set === true ? true : ''
+  env.DEBUG_CHUNKS = set === 'chunker' || set === true ? true : ''
+  return this
+}
+
+// conjugation helpers, usable without a document
+Object.assign(ja, lexMethods)
+
+/** parse a japanese numeral - nlp.toNumber('二十三') === 23 */
+ja.toNumber = toNumber
+
+/** write a number in kanji - nlp.toKanji(23) === '二十三' */
+ja.toKanji = toKanji
+
+ja.version = version
+
+export default ja

package/types/index.d.ts

@@ -0,0 +1,268 @@
+import type {
+  Document, Pointer, Groups, JsonProps, outMethods, Term, Net,
+  Lexicon, Plugin, matchOptions, Match,
+} from './misc'
+import type {
+  VerbClass, VerbConjugation, IAdjectiveConjugation, NaAdjectiveConjugation,
+  Deconjugation, NumberParse,
+} from './japanese'
+
+/** a view onto some part of a parsed document */
+declare class View {
+  // ---- utils ----
+  /** is this document empty? */
+  found: boolean
+  /** the term objects in this view */
+  docs: Document
+  /** the whole parsed document */
+  document: Document
+  /** the indexes for the current view */
+  pointer: Pointer[] | null
+  /** explicit indexes for the current view */
+  fullPointer: Pointer[]
+  /** internal library methods */
+  methods: object
+  /** internal library data */
+  model: object
+  /** which compute methods run by default */
+  hooks: string[]
+  /** helper for detecting a compromise object */
+  isView: boolean
+  /** the number of characters in each match */
+  length: number
+
+  update: (pointer: Pointer | null) => View
+  toView: (pointer: Pointer | null) => View
+  fromText: (text: string) => View
+  /** .docs [alias] */
+  termList: () => Term[]
+  /** run a named analysis - 'root', 'romanji', 'readings', 'english' */
+  compute: (method: string | string[]) => View
+  /** deep-copy the document, so no references remain */
+  clone: (shallow?: boolean) => View
+
+  // ---- loops ----
+  forEach: (fn: (m: View) => void) => View
+  map: (fn: (m: View) => any, emptyResult?: any) => View | []
+  filter: (fn: (m: View) => boolean) => View
+  find: (fn: (m: View) => boolean) => View | undefined
+  some: (fn: (m: View) => boolean) => View
+  random: (n?: number) => View
+
+  // ---- accessors ----
+  terms: (n?: number) => View
+  groups: (name?: string) => View | Groups
+  eq: (n: number) => View
+  first: (n?: number) => View
+  last: (n?: number) => View
+  firstTerms: () => View
+  lastTerms: () => View
+  slice: (start: number, end?: number) => View
+  all: () => View
+  fullSentences: () => View
+  none: () => View
+  isDoc: (view?: View) => boolean
+  wordCount: () => number
+
+  // ---- match ----
+  match: (match: string | View | Net, group?: string | number, options?: matchOptions) => View
+  matchOne: (match: string | View | Net, group?: string | number, options?: matchOptions) => View
+  has: (match: string | View | Net, group?: string | number, options?: matchOptions) => boolean
+  if: (match: string | View | Net, group?: string | number, options?: matchOptions) => View
+  ifNo: (match: string | View | Net, group?: string | number, options?: matchOptions) => View
+  before: (match: string | View, group?: string | number, options?: matchOptions) => View
+  after: (match: string | View, group?: string | number, options?: matchOptions) => View
+  growLeft: (match: string | View, group?: string | number, options?: matchOptions) => View
+  growRight: (match: string | View, group?: string | number, options?: matchOptions) => View
+  grow: (match: string | View, group?: string | number, options?: matchOptions) => View
+  sweep: (match: Net, opts?: object) => { view: View; found: object[] }
+  splitOn: (match?: string, group?: string | number) => View
+  splitBefore: (match?: string, group?: string | number) => View
+  splitAfter: (match?: string, group?: string | number) => View
+  split: (match?: string, group?: string | number) => View
+
+  // ---- case ----
+  toLowerCase: () => View
+  toUpperCase: () => View
+  toTitleCase: () => View
+  toCamelCase: () => View
+
+  // ---- insert ----
+  concat: (input: string | View) => View
+  insertBefore: (input: string | View) => View
+  prepend: (input: string | View) => View
+  insertAfter: (text: string | View) => View
+  append: (text: string | View) => View
+  insert: (text: string | View) => View
+  remove: (match?: string | View) => View
+  delete: (match?: string | View) => View
+  replace: (from: string | View, to?: string | Function, keep?: object) => View
+  replaceWith: (to: string | Function, keep?: object) => View
+  unique: () => View
+  reverse: () => View
+  sort: (method?: string | Function) => View
+
+  // ---- whitespace ----
+  pre: (str?: string, concat?: boolean) => View
+  post: (str?: string, concat?: boolean) => View
+  trim: () => View
+  hyphenate: () => View
+  dehyphenate: () => View
+  deHyphenate: () => View
+  toQuotations: (start?: string, end?: string) => View
+  toQuotation: (start?: string, end?: string) => View
+  toParentheses: (start?: string, end?: string) => View
+
+  // ---- output ----
+  /** the document as text. `'root'` gives dictionary-forms */
+  text: (options?: string | object) => string
+  json: (options?: JsonProps | string) => any
+  /** pretty-print the document and its tags to the console */
+  debug: () => View
+  out: (format?: outMethods) => any
+  html: (toHighlight: object) => string
+  wrap: (matches: object) => string
+
+  // ---- pointers ----
+  union: (match: string | View) => View
+  and: (match: string | View) => View
+  intersection: (match: string | View) => View
+  not: (match: string | View, options?: any) => View
+  difference: (match: string | View, options?: any) => View
+  complement: (match: string | View) => View
+  settle: (match: string | View) => View
+
+  // ---- tags ----
+  tag: (tag: string, reason?: string) => View
+  tagSafe: (tag: string, reason?: string) => View
+  unTag: (tag: string, reason?: string) => View
+  canBe: (tag: string) => View
+
+  // ---- cache ----
+  cache: (options?: object) => View
+  uncache: (options?: object) => View
+  lookup: (trie: object | string[], opts?: object) => View
+  autoFill: () => View
+
+  // ---- japanese ----
+  /** every 動詞 in the document */
+  verbs: () => View
+  /** every 名詞 in the document */
+  nouns: () => View
+  /** every 形容詞 and 形容動詞 in the document */
+  adjectives: () => View
+  /** every 助詞 in the document */
+  particles: () => View
+  /** every number in the document */
+  numbers: (n?: number) => Numbers
+  /** .numbers() alias */
+  values: (n?: number) => Numbers
+  /** every 助数詞 in the document */
+  counters: (n?: number) => View
+  /** every 円/ドル amount */
+  money: (n?: number) => View
+  /** every percentage */
+  percentages: (n?: number) => View
+  /** every date, time and duration */
+  dates: () => View
+  /** the document sounded-out in the latin alphabet */
+  romanji: () => string
+  /** the dictionary-form of each match - 食べました → 食べる */
+  toInfinitive: () => string[]
+}
+
+/** a view of the numbers in a document */
+declare class Numbers extends View {
+  /** the parsed number, and how it was written */
+  parse: (n?: number) => NumberParse[]
+  /** the value of each number - 「二十三」 gives 23 */
+  get: (n?: number) => (number | null)[]
+  /** the 助数詞 attached to each number - 五冊 gives 冊 */
+  units: () => View
+  /** only the ordinals */
+  isOrdinal: () => Numbers
+  /** only the cardinals */
+  isCardinal: () => Numbers
+  /** rewrite each number in digits - 二十三 becomes 23 */
+  toNumber: () => Numbers
+  /** rewrite each number in kanji - 23 becomes 二十三 */
+  toText: () => Numbers
+  /** add thousands-separators - 1234 becomes 1,234 */
+  toLocaleString: () => Numbers
+  /** replace each number, keeping the script it was written in */
+  set: (n: number) => Numbers
+  add: (n: number) => Numbers
+  subtract: (n: number) => Numbers
+  increment: () => Numbers
+  decrement: () => Numbers
+  isEqual: (n: number) => Numbers
+  greaterThan: (n: number) => Numbers
+  lessThan: (n: number) => Numbers
+  between: (min: number, max: number) => Numbers
+}
+
+/** parse some japanese text */
+declare function nlp(text: string, lexicon?: Lexicon): View
+
+declare namespace nlp {
+  // ---- constructor ----
+  /** interpret text without tagging */
+  export function tokenize(text: string, lexicon?: Lexicon): View
+  /** mix-in a compromise plugin */
+  export function plugin(plugin: Plugin): any
+  /** mix-in a compromise plugin */
+  export function extend(plugin: Plugin): any
+  /** turn a match-string into json */
+  export function parseMatch(match: string, opts?: matchOptions): object[]
+  /** grab library internals */
+  export function world(): object
+  /** grab all current linguistic data */
+  export function model(): object
+  /** grab exposed library methods */
+  export function methods(): object
+  /** which compute methods run automatically */
+  export function hooks(): string[]
+  /** log the tagger's decision-making to the console */
+  export function verbose(mode?: boolean | 'tagger' | 'match' | 'chunker'): any
+  /** current semver version of the library */
+  export const version: string
+  /** connect new tags to the tagset graph */
+  export function addTags(tags: object): any
+  /** add new words to the lexicon */
+  export function addWords(words: Lexicon): any
+  /** turn a list of words into a searchable graph */
+  export function buildTrie(words: string[]): object
+  /** compile a set of match objects into a faster form */
+  export function buildNet(matches: Match[]): Net
+  /** add words to the autoFill dictionary */
+  export function typeahead(words: Lexicon): any
+
+  // ---- japanese ----
+  /**
+   * the full paradigm of a dictionary-form verb.
+   * returns null if the word isn't a shape we can conjugate.
+   */
+  export function conjugate(dictionaryForm: string): VerbConjugation | null
+  /**
+   * the paradigm of an adjective. a word ending in い is treated as an
+   * い-adjective, anything else as a な-adjective.
+   */
+  export function conjugateAdjective(word: string): IAdjectiveConjugation | NaAdjectiveConjugation | null
+  /** walk a conjugated verb back to its dictionary-form */
+  export function deconjugate(word: string): Deconjugation | null
+  /** which paradigm a dictionary-form verb follows */
+  export function verbClass(dictionaryForm: string): VerbClass | null
+  /**
+   * parse a japanese numeral - `nlp.toNumber('二十三')` is 23.
+   * handles kanji, half-width and full-width digits.  null if it isn't one.
+   */
+  export function toNumber(numeral: string): number | null
+  /** write a number in kanji - `nlp.toKanji(23)` is 二十三 */
+  export function toKanji(num: number): string | null
+}
+
+export default nlp
+export type {
+  View, Numbers, VerbClass, VerbConjugation, IAdjectiveConjugation,
+  NaAdjectiveConjugation, Deconjugation, NumberParse, Term, Lexicon, Plugin,
+}

package/types/japanese.d.ts

@@ -0,0 +1,159 @@
+// the japanese-specific shapes: conjugation paradigms and verb classes.
+
+/** which paradigm a verb follows */
+export type VerbClass =
+  | 'godan'    // 五段 - 書く, 話す, 帰る
+  | 'ichidan'  // 一段 - 食べる, 見る
+  | 'suru'     // する, and every 〜する compound
+  | 'kuru'     // 来る, and て-form + 来る
+  | 'aru'      // ある - its negative is ない, not あらない
+  | 'iku'      // 行く - its て-form is 行って, not 行いて
+  | 'ou'       // 問う, 請う - 問うて, not 問って
+  | 'aru5'     // くださる, なさる, いらっしゃる - masu-stem drops the り
+
+/** every form of a verb, built from its dictionary-form */
+export interface VerbConjugation {
+  /** which paradigm produced these */
+  class: VerbClass
+  /** 辞書形 - 書く */
+  Infinitive: string
+  /** 連用形, the 'masu-stem' - 書き */
+  Stem: string
+  /** 書く */
+  PresentTense: string
+  /** 書いた */
+  PastTense: string
+  /** 書かない */
+  Negative: string
+  /** 書かなかった */
+  PastNegative: string
+  /** て形 - 書いて */
+  Gerund: string
+  /** 書かなくて */
+  NegativeGerund: string
+  /** 書きます */
+  Polite: string
+  /** 書きました */
+  PolitePast: string
+  /** 書きません */
+  PoliteNegative: string
+  /** 書きませんでした */
+  PolitePastNegative: string
+  /** 書きましょう */
+  PoliteVolitional: string
+  /** 命令形 - 書け */
+  Imperative: string
+  /** 書くな */
+  NegativeImperative: string
+  /** 書こう */
+  Volitional: string
+  /** 可能形 - 書ける */
+  Potential: string
+  /** 受身形 - 書かれる */
+  Passive: string
+  /** 使役形 - 書かせる */
+  Causative: string
+  /** 書かせられる */
+  CausativePassive: string
+  /** 〜たら - 書いたら */
+  Conditional: string
+  /** 〜ば - 書けば */
+  Provisional: string
+  /** 書いている */
+  Progressive: string
+  /** 書いています */
+  ProgressivePolite: string
+  /** 書いていた */
+  PastProgressive: string
+  /** 書きたい */
+  Desire: string
+  /** 〜たり - 書いたり */
+  Representative: string
+  /** 書くだろう */
+  Presumptive: string
+  /** 書きながら */
+  Continuative: string
+}
+
+/** every form of an い-adjective */
+export interface IAdjectiveConjugation {
+  /** 高い */
+  Infinitive: string
+  PresentTense: string
+  /** 高くない */
+  Negative: string
+  /** 高かった */
+  PastTense: string
+  /** 高くなかった */
+  PastNegative: string
+  /** 高くて */
+  Gerund: string
+  /** 高く */
+  Adverb: string
+  /** 高ければ */
+  Provisional: string
+  /** 高かろう */
+  Presumptive: string
+  /** 高いです */
+  Polite: string
+  /** 高かったです */
+  PolitePast: string
+  /** 高くないです */
+  PoliteNegative: string
+  /** 高すぎる */
+  Superlative: string
+  /** 高そう */
+  Impression: string
+  /** 高さ */
+  Nominal: string
+}
+
+/** every form of a な-adjective. it inflects with the copula */
+export interface NaAdjectiveConjugation {
+  /** 静か */
+  Infinitive: string
+  /** 静かな */
+  Adnominal: string
+  /** 静かだ */
+  PresentTense: string
+  /** 静かじゃない */
+  Negative: string
+  /** 静かだった */
+  PastTense: string
+  /** 静かじゃなかった */
+  PastNegative: string
+  /** 静かで */
+  Gerund: string
+  /** 静かに */
+  Adverb: string
+  /** 静かです */
+  Polite: string
+  /** 静かでした */
+  PolitePast: string
+  /** 静かじゃありません */
+  PoliteNegative: string
+  /** 静かなら */
+  Provisional: string
+}
+
+/** the result of walking a conjugated verb back to its dictionary-form */
+export interface Deconjugation {
+  /** the dictionary-form - 書きました → 書く */
+  root: string
+  /** what the ending told us - ['Verb', 'PastTense', 'Polite'] */
+  tags: string[]
+  /** every dictionary-form the ending could have come from */
+  candidates: string[]
+  /** true when no candidate could be confirmed, and `root` is a guess */
+  guess?: boolean
+}
+
+/** what `.numbers().parse()` reports about a number */
+export interface NumberParse {
+  /** the value, or null if it couldn't be read */
+  num: number | null
+  /** written in kanji (二十三) rather than digits */
+  kanji: boolean
+  /** written in full-width digits (２３) */
+  wide: boolean
+}

package/types/misc.d.ts

@@ -0,0 +1,92 @@
+// shared types.
+// compromise ships its own copies of these, but its `./one` subpath has no
+// `types` condition in its exports map, so they can't be imported from here.
+// they're restated rather than re-exported - keep them in sync with
+// compromise/types/misc.ts when bumping the dependency.
+
+export type Document = Term[][]
+
+export type Pointer = [n?: number, start?: number, end?: number, startId?: string, endId?: string]
+
+export type outMethods = 'text' | 'normal' | 'offset' | 'terms' | 'topk' | 'json' | 'tags' | 'array' | 'debug'
+
+export type Groups = object
+
+export interface Term {
+  text: string
+  pre: string
+  post: string
+  normal: string
+  tags?: Set<string>
+  index?: [n?: number, start?: number]
+  id?: string
+  chunk?: string
+  dirty?: boolean
+  /** the dictionary-form, after `.compute('root')` - 食べました → 食べる */
+  root?: string
+  /** latin-alphabet spelling, after `.compute('romanji')` */
+  romanji?: string
+  /** kana spelling of the word's kanji, after `.compute('readings')` */
+  reading?: string
+  /** the parsed value of a numeral - 「二十三」 gives 23 */
+  number?: number
+}
+
+/** options for `.json()` */
+export interface JsonProps {
+  text?: boolean
+  normal?: boolean
+  reduced?: boolean
+  trim?: boolean
+  offset?: boolean
+  count?: boolean
+  unique?: boolean
+  index?: boolean
+  terms?: {
+    text?: boolean
+    normal?: boolean
+    clean?: boolean
+    implicit?: boolean
+    tags?: boolean
+    whitespace?: boolean
+    id?: boolean
+    offset?: boolean
+    bestTag?: boolean
+  }
+}
+
+/** a key-value object of words and their tags */
+export interface Lexicon {
+  [key: string]: string
+}
+
+export interface Plugin {
+  methods?: object
+  model?: object
+  compute?: object
+  hooks?: string[]
+  tags?: object
+  words?: object
+  lib?: () => object
+  api?: (fn: (view: any) => {}) => void
+  mutate?: (fn: (world: object) => {}) => void
+}
+
+export interface matchOptions {
+  fuzzy?: number
+  caseSensitive?: boolean
+}
+
+export interface Match {
+  match: string
+  tag?: string | string[]
+  unTag?: string | string[]
+  group?: string | number
+  reason?: string
+}
+
+export interface Net {
+  hooks: object
+  always?: any
+  isNet: boolean
+}