hanlint 0.0.2 → 0.0.8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (146) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +23 -5
  3. package/data/adjectiveStems.txt +118 -0
  4. package/data/baselineAnchors.txt +3 -0
  5. package/data/blindEvaluation.schema.json +26 -0
  6. package/data/bridgeOpeners.txt +10 -0
  7. package/data/candidateActionNouns.txt +39 -0
  8. package/data/candidateSplitEndings.txt +13 -0
  9. package/data/cliches.json +1 -1
  10. package/data/confusable.json +2 -2
  11. package/data/deixis.txt +10 -3
  12. package/data/draftHistory.txt +10 -0
  13. package/data/easyWords.json +0 -42
  14. package/data/enoughMarkers.txt +6 -0
  15. package/data/entailmentPredictions.schema.json +52 -0
  16. package/data/euiNouns.txt +3 -4
  17. package/data/evidenceEntailmentBenchmark.schema.json +126 -0
  18. package/data/exemplars.json +347 -0
  19. package/data/holeKinds.json +1 -2
  20. package/data/iVerbStems.txt +26 -0
  21. package/data/inNouns.txt +147 -0
  22. package/data/irregularStems.txt +64 -0
  23. package/data/nonNouns.txt +153 -0
  24. package/data/panelAssignmentReview.schema.json +117 -0
  25. package/data/panelJudgePredictions.schema.json +78 -0
  26. package/data/panelReviewBatch.schema.json +79 -0
  27. package/data/panelTrialSet.schema.json +44 -0
  28. package/data/patterns.json +114 -0
  29. package/data/profiles.json +1 -0
  30. package/data/readerCalls.txt +12 -7
  31. package/data/redundantPair.json +1 -1
  32. package/data/rieulStems.txt +30 -0
  33. package/data/ruleCategories.json +56 -0
  34. package/data/ruleDocs.json +29 -17
  35. package/data/ruleMechanisms.json +56 -0
  36. package/data/spacing.json +0 -12
  37. package/data/spelling.json +6 -24
  38. package/data/translationese.json +12 -2
  39. package/data/verbTails.txt +17 -0
  40. package/data/version.json +1 -1
  41. package/data/writingBrief.schema.json +64 -0
  42. package/data/writingBriefV2.schema.json +116 -0
  43. package/data/writingTrial.schema.json +36 -0
  44. package/package.json +13 -5
  45. package/src/analysis/grammar/ending.js +336 -0
  46. package/src/analysis/grammar/hangul.js +94 -0
  47. package/src/analysis/grammar/index.js +5 -0
  48. package/src/analysis/grammar/josa.js +73 -0
  49. package/src/analysis/grammar/register.js +115 -0
  50. package/src/analysis/grammar/voice.js +43 -0
  51. package/src/analysis/index.js +4 -31
  52. package/src/analysis/josa.js +2 -0
  53. package/src/analysis/{surface/splitSentences.js → splitSentences.js} +1 -1
  54. package/src/analysis/{surface/tokenize.js → tokenize.js} +70 -13
  55. package/src/baseline/store.js +114 -0
  56. package/src/cli/main.js +506 -58
  57. package/src/cli/walk.js +30 -0
  58. package/src/cli/welcome.js +52 -0
  59. package/src/config/settings.js +103 -8
  60. package/src/data/exemplars.js +177 -0
  61. package/src/data/load.js +10 -0
  62. package/src/data/operations.js +278 -0
  63. package/src/data/patches.js +125 -0
  64. package/src/data/patterns.js +48 -0
  65. package/src/data/profiles.js +102 -0
  66. package/src/document/model.js +2 -0
  67. package/src/document/parseMarkdown.js +84 -11
  68. package/src/edit/applyFixes.js +0 -0
  69. package/src/fingerprint/build.js +50 -32
  70. package/src/{rules/shared → fingerprint}/codeBlocks.js +11 -11
  71. package/src/fingerprint/codeMarkers.js +73 -0
  72. package/src/fingerprint/markers.js +9 -0
  73. package/src/fingerprint/readerState.js +126 -0
  74. package/src/fingerprint/sourceSentence.js +37 -0
  75. package/src/fingerprint/topics.js +1 -1
  76. package/src/index.js +2 -16
  77. package/src/report/compactReport.js +6 -1
  78. package/src/report/fingerprintJson.js +8 -3
  79. package/src/report/jsonReport.js +50 -3
  80. package/src/report/operationMatch.js +31 -0
  81. package/src/report/patchMatch.js +44 -0
  82. package/src/report/registerMatch.js +40 -0
  83. package/src/report/textReport.js +41 -5
  84. package/src/rules/code/duplicateBlock.js +4 -3
  85. package/src/rules/code/firstResultDistance.js +1 -0
  86. package/src/rules/code/inputFileSource.js +23 -68
  87. package/src/rules/code/installImport.js +4 -4
  88. package/src/rules/code/platformApi.js +7 -6
  89. package/src/rules/document/countMismatch.js +22 -6
  90. package/src/rules/document/enoughOnce.js +25 -0
  91. package/src/rules/document/fieldEcho.js +38 -0
  92. package/src/rules/document/keywordHeading.js +27 -0
  93. package/src/rules/document/keywordMissing.js +1 -0
  94. package/src/rules/document/noQuestion.js +5 -1
  95. package/src/rules/document/promiseRecall.js +4 -2
  96. package/src/rules/document/tableOddCell.js +60 -0
  97. package/src/rules/finding.js +24 -2
  98. package/src/rules/orthography/confusable.js +1 -0
  99. package/src/rules/orthography/spacing.js +1 -0
  100. package/src/rules/orthography/spelling.js +1 -0
  101. package/src/rules/paragraph/factListParagraph.js +1 -0
  102. package/src/rules/paragraph/paraFragment.js +21 -11
  103. package/src/rules/paragraph/topicBreak.js +1 -0
  104. package/src/rules/registry.js +72 -3
  105. package/src/rules/sentence/cliche.js +1 -0
  106. package/src/rules/sentence/connectorRepeat.js +8 -7
  107. package/src/rules/sentence/danglingDeixis.js +8 -3
  108. package/src/rules/sentence/dash.js +2 -1
  109. package/src/rules/sentence/deixis.js +7 -3
  110. package/src/rules/sentence/doubleNegative.js +4 -1
  111. package/src/rules/sentence/doublePassive.js +38 -1
  112. package/src/rules/sentence/draftHistory.js +24 -0
  113. package/src/rules/sentence/ellipsis.js +17 -3
  114. package/src/rules/sentence/endingRepeat.js +6 -2
  115. package/src/rules/sentence/euiChain.js +3 -2
  116. package/src/rules/sentence/fillerOpener.js +1 -0
  117. package/src/rules/sentence/hardWord.js +1 -0
  118. package/src/rules/sentence/imperativePeriod.js +2 -1
  119. package/src/rules/sentence/japaneseLoan.js +1 -0
  120. package/src/rules/sentence/longSentence.js +3 -1
  121. package/src/rules/sentence/negationRedefine.js +10 -1
  122. package/src/rules/sentence/nounPile.js +1 -0
  123. package/src/rules/sentence/numberOrphan.js +45 -0
  124. package/src/rules/sentence/outsideProfile.js +68 -0
  125. package/src/rules/sentence/redundantPair.js +1 -0
  126. package/src/rules/sentence/translationese.js +1 -0
  127. package/src/rules/shared/candidates.js +129 -0
  128. package/src/rules/shared/dictionaryRule.js +12 -2
  129. package/src/rules/shared/localAntecedent.js +12 -0
  130. package/src/rules/shared/repeat.js +48 -0
  131. package/src/rules/structure/blockUnread.js +33 -0
  132. package/src/rules/structure/bridgeRepeat.js +40 -0
  133. package/src/rules/structure/emojiBullet.js +1 -0
  134. package/src/rules/structure/headingQuestion.js +19 -0
  135. package/src/rules/structure/headingSentence.js +9 -4
  136. package/src/rules/structure/headingSkip.js +1 -0
  137. package/src/rules/structure/headingUniform.js +16 -18
  138. package/src/rules/structure/introImage.js +19 -0
  139. package/src/rules/structure/introLong.js +1 -0
  140. package/src/rules/structure/loneSubheading.js +42 -0
  141. package/src/rules/structure/moreLater.js +48 -0
  142. package/src/rules/structure/sectionNoProse.js +1 -0
  143. package/src/rules/structure/sectionResult.js +3 -2
  144. package/src/text.js +19 -0
  145. package/src/rules/document/readerAbsent.js +0 -11
  146. package/src/rules/shared/runs.js +0 -22
@@ -4,15 +4,20 @@
4
4
  * 빈 줄로 블록을 나누고 첫 줄로 종류를 정한다. 코드 펜스 안은 통째로 code 다. H2 가 절을 열고 첫 H2 앞은 도입 절이다.
5
5
  */
6
6
  import { splitLines } from "../text.js";
7
- import { CODE, EMBED, HEADING, HTML, IMAGE, LIST, PROSE, TABLE } from "./model.js";
7
+ import { CODE, EMBED, HEADING, HTML, IMAGE, LIST, PROSE, QUOTE, TABLE } from "./model.js";
8
8
 
9
9
  const FRONTMATTER = /^---\r?\n([\s\S]*?)\r?\n---\r?\n/;
10
- const FENCE = /^\s*(```|~~~)/;
10
+ const FENCE_OPEN = /^\s*(?:(`{3,})([^`]*)|(~{3,})([^~]*))$/;
11
+ const FENCE_CLOSE = /^\s*(`{3,}|~{3,})\s*$/;
11
12
  const HEADING_LINE = /^(#{1,6})\s+(.*?)\s*#*\s*$/;
13
+ const INDENTED_CODE_LINE = /^(?: {4}|\t)/;
14
+ const QUOTE_LINE = /^\s*>/;
12
15
  const IMAGE_LINE = /^!\[/;
13
- const LIST_LINE = /^\s*(?:[-*+]|\d+[.)])\s+/;
16
+ const LIST_LINE = /^(\s*)(?:[-*+]|\d+[.)])\s+/;
14
17
  const TABLE_LINE = /^\s*\|/;
15
18
  const URL_LINE = /^\s*https?:\/\/\S+\s*$/;
19
+ // 링크 하나만 있는 줄은 끼워 넣은 것 (영상 카드, 실행 칸, 참고 링크) 이라 주소만 있는 줄과 같은 embed 다. 파이썬과 같다.
20
+ const LINK_ONLY_LINE = /^\s*\[[^\]\n]+\]\(\S+?(?:\s+"[^"\n]*")?\)\s*$/;
16
21
  const HTML_LINE = /^\s*</;
17
22
  const META_KEY = /^[A-Za-z][A-Za-z0-9_]*$/;
18
23
  const CONTROL = /^\s*<!--\s*hanlint-(disable-next-line|disable-next|disable|enable)\b([^>]*?)-->\s*$/;
@@ -34,16 +39,40 @@ export function parseFrontmatter(text) {
34
39
  }
35
40
 
36
41
  /** @param {string} firstLine */
37
- export function classify(firstLine) {
42
+ export function classify(firstLine, lineCount = 1) {
43
+ if (INDENTED_CODE_LINE.test(firstLine)) return CODE;
38
44
  if (HEADING_LINE.test(firstLine)) return HEADING;
45
+ if (QUOTE_LINE.test(firstLine)) return QUOTE;
39
46
  if (IMAGE_LINE.test(firstLine)) return IMAGE;
40
47
  if (TABLE_LINE.test(firstLine)) return TABLE;
41
48
  if (LIST_LINE.test(firstLine)) return LIST;
42
- if (URL_LINE.test(firstLine)) return EMBED;
49
+ if (URL_LINE.test(firstLine) || (lineCount === 1 && LINK_ONLY_LINE.test(firstLine))) return EMBED;
43
50
  if (HTML_LINE.test(firstLine)) return HTML;
44
51
  return PROSE;
45
52
  }
46
53
 
54
+ /** @param {string} line */
55
+ function indentWidth(line) {
56
+ let width = 0;
57
+ for (const char of line) {
58
+ if (char === " ") width += 1;
59
+ else if (char === "\t") width += 4 - (width % 4);
60
+ else break;
61
+ }
62
+ return width;
63
+ }
64
+
65
+ /** @param {string} line @param {number} width */
66
+ function afterIndent(line, width) {
67
+ let index = 0;
68
+ let consumed = 0;
69
+ while (index < line.length && consumed < width && (line[index] === " " || line[index] === "\t")) {
70
+ consumed += line[index] === " " ? 1 : 4 - (consumed % 4);
71
+ index += 1;
72
+ }
73
+ return line.slice(index);
74
+ }
75
+
47
76
  /**
48
77
  * @param {string} text
49
78
  * @param {number} firstLine
@@ -61,10 +90,18 @@ export function splitBlocks(text, firstLine) {
61
90
  let fenceStart = 0;
62
91
  /** @type {string[]} */
63
92
  let fenceLines = [];
93
+ /** @type {number | null} */
94
+ let listContinuation = null;
95
+ /** @type {number | null} */
96
+ let bufferListContinuation = null;
64
97
 
65
98
  const flush = () => {
66
99
  if (!buffer.length) return;
67
- const kind = classify(buffer[0]);
100
+ let firstLine = buffer[0];
101
+ if (bufferListContinuation !== null && indentWidth(firstLine) >= bufferListContinuation) {
102
+ firstLine = afterIndent(firstLine, bufferListContinuation);
103
+ }
104
+ const kind = classify(firstLine, buffer.length);
68
105
  let joined = buffer.join("\n");
69
106
  let level = 0;
70
107
  if (kind === HEADING) {
@@ -74,6 +111,7 @@ export function splitBlocks(text, firstLine) {
74
111
  }
75
112
  blocks.push({ kind, startLine: bufferStart, endLine: bufferStart + buffer.length - 1, text: joined, level, index: blocks.length });
76
113
  buffer = [];
114
+ bufferListContinuation = null;
77
115
  };
78
116
 
79
117
  for (const raw of splitLines(text)) {
@@ -81,17 +119,18 @@ export function splitBlocks(text, firstLine) {
81
119
  const line = raw.replace(/\r+$/, "");
82
120
  if (fence) {
83
121
  fenceLines.push(line);
84
- if (FENCE.test(line) && line.trim().startsWith(fence)) {
122
+ const closing = FENCE_CLOSE.exec(line);
123
+ if (closing && closing[1].startsWith(fence) && closing[1][0] === fence[0]) {
85
124
  blocks.push({ kind: CODE, startLine: fenceStart, endLine: lineNo, text: fenceLines.join("\n"), level: 0, index: blocks.length });
86
125
  fence = null;
87
126
  fenceLines = [];
88
127
  }
89
128
  continue;
90
129
  }
91
- const opening = FENCE.exec(line);
130
+ const opening = FENCE_OPEN.exec(line);
92
131
  if (opening) {
93
132
  flush();
94
- fence = opening[1];
133
+ fence = opening[1] || opening[3];
95
134
  fenceStart = lineNo;
96
135
  fenceLines = [line];
97
136
  continue;
@@ -106,7 +145,17 @@ export function splitBlocks(text, firstLine) {
106
145
  continue;
107
146
  }
108
147
  if (HEADING_LINE.test(line) && buffer.length) flush();
109
- if (!buffer.length) bufferStart = lineNo;
148
+ const listMatch = LIST_LINE.exec(line);
149
+ if (listMatch) {
150
+ const contentColumn = listMatch[0].length;
151
+ listContinuation = Math.ceil(contentColumn / 4) * 4;
152
+ } else if (listContinuation !== null && indentWidth(line) < listContinuation) {
153
+ listContinuation = null;
154
+ }
155
+ if (!buffer.length) {
156
+ bufferStart = lineNo;
157
+ bufferListContinuation = listContinuation;
158
+ }
110
159
  buffer.push(line);
111
160
  if (HEADING_LINE.test(line)) flush();
112
161
  }
@@ -170,6 +219,30 @@ export function disabledRanges(blocks) {
170
219
  return ranges;
171
220
  }
172
221
 
222
+ /** 코드 블록 첫 줄의 언어 표기 (```python 의 python). 없으면 빈 문자열. 소문자로 준다. @param {string} blockText */
223
+ export function fenceLanguage(blockText) {
224
+ const opening = FENCE_OPEN.exec(blockText.split("\n", 1)[0]);
225
+ if (!opening) return "";
226
+ const info = (opening[2] || opening[4] || "").trim();
227
+ return info ? info.split(/\s+/)[0].toLowerCase() : "";
228
+ }
229
+
230
+ /**
231
+ * 언어 표기가 languages 에 든 펜스를 없앤 문서. 블록 순서 번호를 다시 매기고 절을 다시 묶는다. 인라인 제어는 원문 줄
232
+ * 기준이라 그대로 두고, 뺀 펜스는 ignored 에 남긴다. 파이썬 dropFences 와 같다.
233
+ * @param {import("./model.js").Document} doc
234
+ * @param {string[]} languages
235
+ * @returns {import("./model.js").Document}
236
+ */
237
+ export function dropFences(doc, languages) {
238
+ const wanted = new Set(languages.map((name) => name.trim().toLowerCase()).filter(Boolean));
239
+ if (!wanted.size) return doc;
240
+ const dropped = doc.blocks.filter((block) => block.kind === CODE && wanted.has(fenceLanguage(block.text)));
241
+ if (!dropped.length) return doc;
242
+ const blocks = doc.blocks.filter((block) => !dropped.includes(block)).map((block, index) => ({ ...block, index }));
243
+ return { path: doc.path, frontmatter: doc.frontmatter, blocks, sections: groupSections(blocks), disabled: doc.disabled, ignored: [...doc.ignored, ...dropped] };
244
+ }
245
+
173
246
  /**
174
247
  * @param {string} text
175
248
  * @param {string | null} [path]
@@ -179,5 +252,5 @@ export function parseMarkdown(text, path = null) {
179
252
  const { meta, firstLine } = parseFrontmatter(text);
180
253
  const body = firstLine === 1 ? text : splitLines(text).slice(firstLine - 1).join("\n");
181
254
  const blocks = splitBlocks(body, firstLine);
182
- return { path, frontmatter: meta, blocks, sections: groupSections(blocks), disabled: disabledRanges(blocks) };
255
+ return { path, frontmatter: meta, blocks, sections: groupSections(blocks), disabled: disabledRanges(blocks), ignored: [] };
183
256
  }
Binary file
@@ -1,11 +1,16 @@
1
1
  // @ts-check
2
2
  /** 문서 모델을 한 번 훑어 글 지문을 만든다. 텍스트를 읽는 유일한 자리다. 파이썬 fingerprint/build.py 와 같다. */
3
3
  import { defaultConfig } from "../config/settings.js";
4
+ import { NONE as NO_REGISTER, documentRegister, lastWord, registerOfWord } from "../analysis/grammar/register.js";
5
+ import { doublePassiveSpans, euiCount, longestNounRun, splitSentences } from "../analysis/index.js";
4
6
  import { HEADING, PROSE, sectionStartLine, sectionTitle } from "../document/model.js";
7
+ import { dropFences } from "../document/parseMarkdown.js";
5
8
  import { codeSpans, plainText } from "../document/plainText.js";
6
9
  import { countIn, mean, pstdev, wordCount } from "../text.js";
10
+ import { codeBlocksOf } from "./codeBlocks.js";
7
11
  import { entriesFor, matchesIn } from "./dictionaries.js";
8
12
  import * as markers from "./markers.js";
13
+ import { buildReaderTrail } from "./readerState.js";
9
14
  import { overlap, topicsOf, unionOf } from "./topics.js";
10
15
 
11
16
  /**
@@ -19,6 +24,7 @@ import { overlap, topicsOf, unionOf } from "./topics.js";
19
24
  * @property {number} length 어절 수
20
25
  * @property {string} ending
21
26
  * @property {string} mood
27
+ * @property {string} register
22
28
  * @property {number} commas
23
29
  * @property {string | null} connectorStart
24
30
  * @property {number} causal
@@ -81,16 +87,19 @@ import { overlap, topicsOf, unionOf } from "./topics.js";
81
87
  * @property {number} questionCount
82
88
  * @property {number} readerCallCount
83
89
  * @property {[number, string, number, string][]} countPromises (수, 단위, 줄, 원문)
84
- * @property {[number, string][]} promises
85
- * @property {[number, string][]} recalls
90
+ * @property {import("./codeBlocks.js").CodeBlock[]} codeBlocks 코드 펜스를 언어와 본문 줄로 푼 것
91
+ * @property {import("./readerState.js").ReaderTrail} reader 자리마다의 독자 상태. 약속과 회수 표지는 final 에 있다
86
92
  * @property {[string, number, number][]} disabled
93
+ * @property {import("../document/model.js").Block[]} ignored 설정이 지문에서 뺀 펜스
94
+ * @property {string} register
95
+ * @property {number} registerShare
87
96
  * @property {SectionPrint} intro
88
97
  * @property {SectionPrint[]} bodySections
89
98
  */
90
99
 
91
100
  /**
92
101
  * 문장 안의 인용 구간. 블록의 인라인 코드 구간을 문장 좌표로 옮기고 따옴표 쌍을 더한다.
93
- * @param {import("../analysis/surface/splitSentences.js").Sentence} sentence
102
+ * @param {import("../analysis/splitSentences.js").Sentence} sentence
94
103
  * @param {[number, number][]} blockCodeSpans
95
104
  * @returns {[number, number][]}
96
105
  */
@@ -114,35 +123,39 @@ function quotedIn(sentence, blockCodeSpans) {
114
123
  * @param {import("../document/model.js").Block} block
115
124
  * @param {number} paragraphIndex
116
125
  * @param {number} sectionIndex
117
- * @param {import("../analysis/index.js").Analyzer} analyzer
118
126
  * @param {import("./dictionaries.js").Entry[]} entries
119
127
  * @param {[number, number][]} quoted
120
128
  * @returns {SentencePrint}
121
129
  */
122
- function makeSentencePrint(text, index, line, block, paragraphIndex, sectionIndex, analyzer, entries, quoted) {
130
+ function makeSentencePrint(build, text, line, block, sectionIndex, quoted) {
131
+ const { entries } = build;
123
132
  const ending = markers.endingOf(text);
124
133
  const deixis = markers
125
134
  .matchedSpans(text, "deixis.txt")
126
135
  .filter(([start, end]) => !markers.insideAny(start, end, quoted))
127
136
  .map(([, , found]) => found);
128
137
  const matches = matchesIn(text, entries).filter((m) => !markers.insideAny(m.start, m.end, quoted));
138
+ const mood = markers.moodOf(text, ending);
129
139
  return {
130
- index,
140
+ index: build.sentences.length,
131
141
  line,
132
142
  text,
133
143
  blockIndex: block.index,
134
- paragraphIndex,
144
+ paragraphIndex: build.paragraphs.length,
135
145
  sectionIndex,
136
146
  length: wordCount(text),
137
147
  ending,
138
- mood: markers.moodOf(text, ending),
148
+ mood,
149
+ register: mood === "평서" ? (registerOfWord(lastWord(text)) ?? NO_REGISTER) : NO_REGISTER,
139
150
  commas: markers.countCommas(text),
140
151
  connectorStart: markers.connectorStartOf(text),
141
152
  causal: markers.countMatches(text, "causalMarkers.txt"),
142
153
  deixis,
143
- euiCount: analyzer.euiCount(text),
144
- nounRun: analyzer.longestNounRun(text),
145
- passives: analyzer.doublePassives(text),
154
+ euiCount: euiCount(text),
155
+ nounRun: longestNounRun(text),
156
+ passives: doublePassiveSpans(text)
157
+ .filter(([start, end]) => !markers.insideAny(start, end, quoted))
158
+ .map(([, , surface]) => surface),
146
159
  hedges: markers.countMatches(text, "hedges.txt"),
147
160
  numbers: markers.countNumbers(text),
148
161
  topics: topicsOf(text),
@@ -164,11 +177,11 @@ function makeSentencePrint(text, index, line, block, paragraphIndex, sectionInde
164
177
  * @param {boolean} followsProseDirectly
165
178
  * @returns {ParagraphPrint}
166
179
  */
167
- function makeParagraphPrint(sentences, index, block, sectionIndex, previous, followsProseDirectly) {
180
+ function makeParagraphPrint(build, sentences, block, sectionIndex, previous, followsProseDirectly) {
168
181
  const lengths = sentences.map((s) => s.length);
169
182
  const topics = unionOf(sentences.map((s) => s.topics));
170
183
  return {
171
- index,
184
+ index: build.paragraphs.length,
172
185
  blockIndex: block.index,
173
186
  sectionIndex,
174
187
  startLine: block.startLine,
@@ -189,13 +202,13 @@ function makeParagraphPrint(sentences, index, block, sectionIndex, previous, fol
189
202
  /**
190
203
  * @param {import("../document/model.js").Section} section
191
204
  * @param {number} sectionIndex
192
- * @param {import("../analysis/index.js").Analyzer} analyzer
193
205
  * @param {import("./dictionaries.js").Entry[]} entries
194
206
  * @param {SentencePrint[]} sentences
195
207
  * @param {ParagraphPrint[]} paragraphs
196
208
  * @returns {SectionPrint}
197
209
  */
198
- function buildSection(section, sectionIndex, analyzer, entries, sentences, paragraphs) {
210
+ function buildSection(build, section, sectionIndex) {
211
+ const { sentences, paragraphs } = build;
199
212
  /** @type {ParagraphPrint[]} */
200
213
  const sectionParagraphs = [];
201
214
  /** @type {import("../document/model.js").Block | null} */
@@ -209,19 +222,18 @@ function buildSection(section, sectionIndex, analyzer, entries, sentences, parag
209
222
  const blockCodeSpans = codeSpans(block.text, text);
210
223
  /** @type {SentencePrint[]} */
211
224
  const blockSentences = [];
212
- for (const sentence of analyzer.sentences(text)) {
225
+ for (const sentence of splitSentences(text)) {
213
226
  const line = block.startLine + countIn(text, "\n", sentence.start);
214
- const index = sentences.length + blockSentences.length;
215
227
  const quoted = quotedIn(sentence, blockCodeSpans);
216
- blockSentences.push(
217
- makeSentencePrint(sentence.text, index, line, block, paragraphs.length, sectionIndex, analyzer, entries, quoted),
218
- );
228
+ const made = makeSentencePrint(build, sentence.text, line, block, sectionIndex, quoted);
229
+ // 만든 자리에서 바로 쌓는다. 다음 문장의 index 가 이 길이에서 나오므로 미루면 셈이 어긋난다.
230
+ sentences.push(made);
231
+ blockSentences.push(made);
219
232
  }
220
- sentences.push(...blockSentences);
221
233
  const previous = sectionParagraphs.length ? sectionParagraphs[sectionParagraphs.length - 1] : null;
222
234
  const paragraph = makeParagraphPrint(
235
+ build,
223
236
  blockSentences,
224
- paragraphs.length,
225
237
  block,
226
238
  sectionIndex,
227
239
  previous,
@@ -246,23 +258,26 @@ function buildSection(section, sectionIndex, analyzer, entries, sentences, parag
246
258
 
247
259
  /**
248
260
  * @param {import("../document/model.js").Document} doc
249
- * @param {import("../analysis/index.js").Analyzer} analyzer
250
261
  * @param {import("../config/settings.js").Config} [config]
251
262
  * @returns {DocumentPrint}
252
263
  */
253
- export function buildFingerprint(doc, analyzer, config = defaultConfig()) {
254
- const entries = entriesFor(config);
255
- /** @type {SentencePrint[]} */
256
- const sentences = [];
257
- /** @type {ParagraphPrint[]} */
258
- const paragraphs = [];
264
+ export function buildFingerprint(doc, config = defaultConfig()) {
265
+ // 코드도 산문도 아닌 펜스는 지문에 들어오기 전에 뺀다. 모든 진입점이 여기를 지나므로 한 자리면 된다.
266
+ doc = dropFences(doc, config.ignoreFences);
267
+ const build = { entries: entriesFor(config), sentences: /** @type {SentencePrint[]} */ ([]), paragraphs: /** @type {ParagraphPrint[]} */ ([]) };
268
+ const { sentences, paragraphs } = build;
259
269
  const sections = doc.sections.map((section, index) =>
260
- buildSection(section, index, analyzer, entries, sentences, paragraphs),
270
+ buildSection(build, section, index),
261
271
  );
262
272
  const headingBlocks = doc.blocks.filter((b) => b.kind === HEADING);
263
273
  /** @type {[number, string, number][]} */
264
274
  const headings = headingBlocks.map((b) => [b.level, b.text, b.startLine]);
265
275
  const headingQuestions = headingBlocks.filter((b) => b.text.includes("?")).length;
276
+ const [register, registerShare] = documentRegister(
277
+ sentences.filter((sentence) => sentence.mood === "평서").map((sentence) => lastWord(sentence.text)),
278
+ config.registerMinShare,
279
+ );
280
+ const codeBlocks = codeBlocksOf(doc.blocks);
266
281
  return {
267
282
  path: doc.path,
268
283
  frontmatter: { ...doc.frontmatter },
@@ -275,9 +290,12 @@ export function buildFingerprint(doc, analyzer, config = defaultConfig()) {
275
290
  questionCount: sentences.filter((s) => s.mood === "의문").length + headingQuestions,
276
291
  readerCallCount: sentences.filter((s) => s.readerCall || s.mood === "명령").length,
277
292
  countPromises: sentences.flatMap((s) => s.countPromises.map(([n, unit, text]) => [n, unit, s.line, text])),
278
- promises: sentences.flatMap((s) => s.promises.map((text) => [s.line, text])),
279
- recalls: sentences.flatMap((s) => s.recalls.map((text) => [s.line, text])),
293
+ codeBlocks,
294
+ reader: buildReaderTrail(doc.blocks, codeBlocks, sentences),
295
+ register,
296
+ registerShare,
280
297
  disabled: doc.disabled,
298
+ ignored: doc.ignored,
281
299
  intro: sections[0],
282
300
  bodySections: sections.filter((s) => !s.isIntro),
283
301
  };
@@ -1,11 +1,12 @@
1
1
  // @ts-check
2
2
  /**
3
- * 코드 블록을 언어와 본문으로 나눈다. code 부류 규칙이 함께 쓴다. 파이썬 rules/shared/codeBlocks.py 와 같다.
3
+ * 코드 블록을 언어와 본문으로 나눈다. 지문이 한 번 만들고 (DocumentPrint.codeBlocks) code 부류 규칙과 독자 상태가
4
+ * 읽는다. 파이썬 fingerprint/codeBlocks.py 와 같다.
4
5
  * 펜스 첫 줄의 언어 표기 (```python) 를 읽고 본문 줄에 원문 줄 번호를 붙인다. text 펜스는 출력이다.
5
6
  */
6
- import { CODE } from "../../document/model.js";
7
+ import { CODE } from "../document/model.js";
8
+ import { fenceLanguage } from "../document/parseMarkdown.js";
7
9
 
8
- const FENCE_LANGUAGE = /^\s*(?:```|~~~)\s*([A-Za-z0-9_+-]*)/;
9
10
  const CLOSING_FENCE = /^\s*(?:```|~~~)\s*$/;
10
11
 
11
12
  /**
@@ -18,20 +19,19 @@ const CLOSING_FENCE = /^\s*(?:```|~~~)\s*$/;
18
19
  * @property {boolean} isOutput
19
20
  */
20
21
 
21
- /** @param {import("../../fingerprint/build.js").DocumentPrint} doc @returns {CodeBlock[]} */
22
- export function codeBlocksOf(doc) {
22
+ /** @param {import("../document/model.js").Block[]} blocks @returns {CodeBlock[]} */
23
+ export function codeBlocksOf(blocks) {
23
24
  /** @type {CodeBlock[]} */
24
- const blocks = [];
25
- for (const block of doc.blocks) {
25
+ const found = [];
26
+ for (const block of blocks) {
26
27
  if (block.kind !== CODE) continue;
27
28
  const raw = block.text.split("\n");
28
- const match = FENCE_LANGUAGE.exec(raw[0]);
29
- const language = (match ? match[1] : "").toLowerCase();
29
+ const language = fenceLanguage(block.text);
30
30
  let body = raw.slice(1);
31
31
  if (body.length && CLOSING_FENCE.test(body[body.length - 1])) body = body.slice(0, -1);
32
32
  /** @type {[number, string][]} */
33
33
  const lines = body.map((line, offset) => [block.startLine + 1 + offset, line]);
34
- blocks.push({
34
+ found.push({
35
35
  index: block.index,
36
36
  startLine: block.startLine,
37
37
  language,
@@ -40,5 +40,5 @@ export function codeBlocksOf(doc) {
40
40
  isOutput: ["text", "", "output", "console"].includes(language),
41
41
  });
42
42
  }
43
- return blocks;
43
+ return found;
44
44
  }
@@ -0,0 +1,73 @@
1
+ // @ts-check
2
+ /**
3
+ * 코드 줄에서 파일의 공급과 수요를 읽는다. 어떤 줄이 파일을 만들고 (쓰기, 폴더 만들기) 어떤 줄이 파일을 읽는가.
4
+ * 파이썬 fingerprint/codeMarkers.py 와 같다. 줄마다 대조하므로 여러 줄에 걸친 호출은 보지 않는다.
5
+ */
6
+
7
+ const QUOTED = "\\(\\s*[\"']([^\"'\\n]+)[\"']";
8
+ const READS = new RegExp(
9
+ "\\b(?:read_csv|read_excel|read_parquet|read_json|read_sql|read_table|read_ndjson|scan_csv|scan_parquet|scan_ndjson|" +
10
+ "read_text|read_bytes|loadtxt|imread|load_workbook)" + QUOTED,
11
+ "g",
12
+ );
13
+ const OPENS = new RegExp("\\bopen" + QUOTED + "\\s*(?:,\\s*(?:mode\\s*=\\s*)?[\"']([^\"']*)[\"'])?", "g");
14
+ const WRITES = new RegExp(
15
+ "\\b(?:to_csv|to_excel|to_parquet|to_json|to_sql|write_csv|write_parquet|write_json|write_ndjson|sink_parquet|sink_csv|" +
16
+ "write_text|write_bytes|savefig|save|imwrite|savetxt|dump)" + QUOTED,
17
+ "g",
18
+ );
19
+ const SHELL_WRITES = /(?:>>?|-o|-O|--output|Out-File)\s*["']?([\w./\\-]+\.[A-Za-z0-9]+)/g;
20
+ const SQL_PATHS = /FROM\s+'([^']+\.(?:csv|parquet|json|xlsx))'/gi;
21
+ const MAKES_DIR = /(?:mkdir(?:\s+-p)?|makedirs|md)\s*\(?\s*["']?([\w./\\-]+)|Path\(\s*["']([^"']+)["']\s*\)\.mkdir/g;
22
+ /** open 의 모드에 이 글자가 있으면 쓰기다. 없으면 (기본 r) 읽기다. */
23
+ const WRITE_MODES = [..."wax"];
24
+
25
+ /** @param {string} path */
26
+ export function fileName(path) {
27
+ const parts = path.replace(/\\/g, "/").replace(/\/+$/, "").split("/");
28
+ return parts[parts.length - 1];
29
+ }
30
+
31
+ /** @param {string | undefined} mode */
32
+ function isWriteMode(mode) {
33
+ return WRITE_MODES.some((flag) => (mode ?? "r").includes(flag));
34
+ }
35
+
36
+ /** @param {RegExp} pattern @param {string} text @param {number} group */
37
+ function all(pattern, text, group) {
38
+ const found = [];
39
+ for (const match of text.matchAll(pattern)) if (match[group] !== undefined) found.push(match[group]);
40
+ return found;
41
+ }
42
+
43
+ /**
44
+ * 코드 줄들이 만드는 [파일 이름, 폴더 이름]. 쓰기 함수, 셸 리다이렉션, 쓰기 모드의 open, mkdir 을 본다.
45
+ * @param {string[]} lines
46
+ * @returns {[Set<string>, Set<string>]}
47
+ */
48
+ export function createdIn(lines) {
49
+ /** @type {Set<string>} */
50
+ const files = new Set();
51
+ /** @type {Set<string>} */
52
+ const dirs = new Set();
53
+ for (const code of lines) {
54
+ for (const path of all(WRITES, code, 1)) files.add(fileName(path));
55
+ for (const path of all(SHELL_WRITES, code, 1)) files.add(fileName(path));
56
+ for (const match of code.matchAll(OPENS)) if (isWriteMode(match[2])) files.add(fileName(match[1]));
57
+ for (const match of code.matchAll(MAKES_DIR)) dirs.add((match[1] ?? match[2]).replace(/\\/g, "/").split("/")[0]);
58
+ }
59
+ return [files, dirs];
60
+ }
61
+
62
+ /** 한 줄이 읽는 경로. 읽기 함수, 읽기 모드의 open, SQL 의 FROM 차례다. @param {string} code */
63
+ export function readsIn(code) {
64
+ const reads = all(READS, code, 1);
65
+ for (const match of code.matchAll(OPENS)) if (!isWriteMode(match[2])) reads.push(match[1]);
66
+ reads.push(...all(SQL_PATHS, code, 1));
67
+ return reads;
68
+ }
69
+
70
+ /** 한 줄이 쓰기 함수로 쓰는 경로 그대로. 그 경로의 폴더가 있는지 볼 때 쓴다. @param {string} code */
71
+ export function writeTargets(code) {
72
+ return all(WRITES, code, 1);
73
+ }
@@ -8,6 +8,8 @@ import { compile, escape } from "../regex.js";
8
8
 
9
9
  const TRAILING = /[\s.?!"'”’)\]]+$/;
10
10
  const NUMBER = /\d+(?:[.,]\d+)*/g;
11
+ /** 값으로 견주는 수. NUMBER 는 수의 개수를 세고 이것은 수의 정체를 센다. 천 단위 쉼표까지 한 수로 읽는다. */
12
+ const NUMERAL = /\d[\d,]*(?:\.\d+)?/g;
11
13
  const COMMA = /,/g;
12
14
  // 따옴표 쌍. 안은 사용이 아니라 인용이다. 작은따옴표는 앞이 문장 첫머리나 공백이나 여는 괄호일 때만.
13
15
  const QUOTE_PAIRS = [
@@ -165,6 +167,13 @@ export function countPromisesIn(text) {
165
167
  }
166
168
 
167
169
  /** @param {string} text */
170
+ /** 글에 나온 수의 정체 (천 단위 쉼표를 뗀 꼴). 독자 상태가 쌓고 numberOrphan 이 견준다. @param {string} text @returns {Set<string>} */
171
+ export function numeralsIn(text) {
172
+ const found = new Set();
173
+ for (const match of text.matchAll(NUMERAL)) found.add(match[0].replaceAll(",", ""));
174
+ return found;
175
+ }
176
+
168
177
  export function countNumbers(text) {
169
178
  return [...text.matchAll(NUMBER)].length;
170
179
  }
@@ -0,0 +1,126 @@
1
+ // @ts-check
2
+ /**
3
+ * 독자 상태. 글을 위에서 아래로 읽는 독자가 어느 자리에서 무엇을 손에 들고 무엇을 이미 보았는가.
4
+ * 파이썬 fingerprint/readerState.py 와 같다. reader 기제의 규칙은 어느 자리가 요구하는 것을 그 자리의 상태에
5
+ * 대 본다. 상태는 블록과 문장 순서로 쌓이고 한 번 쌓은 것은 바뀌지 않는다. 집합은 고치지 않는다.
6
+ */
7
+ import { CODE, PROSE } from "../document/model.js";
8
+ import { createdIn } from "./codeMarkers.js";
9
+ import { numeralsIn } from "./markers.js";
10
+
11
+ /**
12
+ * 한 자리 (문장 하나 또는 블록 하나) 를 읽기 직전의 독자.
13
+ * @typedef {object} ReaderState
14
+ * @property {import("./build.js").SentencePrint | null} previous 바로 앞에 읽은 산문 문장
15
+ * @property {Set<string>} recent 손에 든 화제어. 바로 앞 문장의 것이고 앞 문장이 없으면 빈 집합
16
+ * @property {Set<string>} known 이 자리 앞까지 산문에 나온 화제어 전부
17
+ * @property {Set<string>} numerals 지금까지 산문과 블록에 나온 수 (천 단위 쉼표를 뗀 꼴)
18
+ * @property {Set<string>} files 앞선 코드 블록이 만든 파일 이름과 폴더
19
+ * @property {number} sentencesRead 지금까지 읽은 산문 문장 수
20
+ * @property {[number, string][]} promises 지금까지 뒤로 미룬 표지 (줄, 원문)
21
+ * @property {[number, string][]} recalls 지금까지 앞을 회수한 표지 (줄, 원문)
22
+ */
23
+
24
+ /**
25
+ * 자리마다의 독자 상태. 문장은 index 로, 블록은 index 로 찾는다.
26
+ * @typedef {object} ReaderTrail
27
+ * @property {ReaderState[]} beforeSentence
28
+ * @property {ReaderState[]} beforeBlock
29
+ * @property {ReaderState} final 글을 다 읽은 독자
30
+ * @property {(blockIndex: number, name: string) => boolean} mentionedBefore 블록보다 앞선 산문이 그 이름을 불렀는가
31
+ */
32
+
33
+ /** @type {Set<string>} */
34
+ const NOTHING = new Set();
35
+
36
+ /**
37
+ * 현재 문장의 화제가 직전 독자에게 새 것인지 이미 손에 든 것인지 네 꼴로 줄인다.
38
+ * @param {import("./build.js").SentencePrint} sentence
39
+ * @param {ReaderState} state
40
+ */
41
+ export function readerKind(sentence, state) {
42
+ if (!sentence.topics.size) return "none";
43
+ if ([...sentence.topics].some((topic) => state.recent.has(topic))) return "recent";
44
+ if ([...sentence.topics].some((topic) => state.known.has(topic))) return "known";
45
+ return "new";
46
+ }
47
+
48
+ /** @param {Set<string>} base @param {Set<string>} added */
49
+ function union(base, added) {
50
+ if (!added.size) return base;
51
+ const out = new Set(base);
52
+ for (const value of added) out.add(value);
53
+ return out;
54
+ }
55
+
56
+ /** @param {ReaderState} state @param {import("./build.js").SentencePrint} sentence @returns {ReaderState} */
57
+ function afterSentence(state, sentence) {
58
+ return {
59
+ previous: sentence,
60
+ recent: sentence.topics,
61
+ known: union(state.known, sentence.topics),
62
+ numerals: union(state.numerals, numeralsIn(sentence.text)),
63
+ files: state.files,
64
+ sentencesRead: state.sentencesRead + 1,
65
+ promises: [...state.promises, ...sentence.promises.map((text) => /** @type {[number, string]} */ ([sentence.line, text]))],
66
+ recalls: [...state.recalls, ...sentence.recalls.map((text) => /** @type {[number, string]} */ ([sentence.line, text]))],
67
+ };
68
+ }
69
+
70
+ /**
71
+ * 산문 아닌 블록을 지난 독자. 수는 어느 블록에서든 보고 파일은 코드 블록이 만든다.
72
+ * @param {ReaderState} state
73
+ * @param {import("../document/model.js").Block} block
74
+ * @param {import("./codeBlocks.js").CodeBlock | undefined} code
75
+ * @returns {ReaderState}
76
+ */
77
+ function afterBlock(state, block, code) {
78
+ let files = state.files;
79
+ if (code) {
80
+ const [made, dirs] = createdIn(code.lines.map(([, line]) => line));
81
+ files = union(union(files, made), dirs);
82
+ }
83
+ return { ...state, numerals: union(state.numerals, numeralsIn(block.text)), files };
84
+ }
85
+
86
+ /**
87
+ * 블록 순서로 한 번 지나며 자리마다의 상태를 적는다. 산문 블록은 문장 하나씩, 나머지는 블록째 읽는다.
88
+ * @param {import("../document/model.js").Block[]} blocks
89
+ * @param {import("./codeBlocks.js").CodeBlock[]} codeBlocks
90
+ * @param {import("./build.js").SentencePrint[]} sentences
91
+ * @returns {ReaderTrail}
92
+ */
93
+ export function buildReaderTrail(blocks, codeBlocks, sentences) {
94
+ const codeByIndex = new Map(codeBlocks.map((code) => [code.index, code]));
95
+ /** @type {ReaderState[]} */
96
+ const beforeSentence = [];
97
+ /** @type {ReaderState[]} */
98
+ const beforeBlock = [];
99
+ /** @type {ReaderState} */
100
+ let state = { previous: null, recent: NOTHING, known: NOTHING, numerals: NOTHING, files: NOTHING, sentencesRead: 0, promises: [], recalls: [] };
101
+ let position = 0;
102
+ for (const block of blocks) {
103
+ beforeBlock.push(state);
104
+ if (block.kind === PROSE) {
105
+ while (position < sentences.length && sentences[position].blockIndex === block.index) {
106
+ beforeSentence.push(state);
107
+ state = afterSentence(state, sentences[position]);
108
+ position += 1;
109
+ }
110
+ } else {
111
+ state = afterBlock(state, block, block.kind === CODE ? codeByIndex.get(block.index) : undefined);
112
+ }
113
+ }
114
+ if (position !== sentences.length) {
115
+ throw new Error(`문장 ${sentences.length - position}개가 어느 블록에도 안 든다. 문장과 블록의 순서가 어긋났다`);
116
+ }
117
+ return {
118
+ beforeSentence,
119
+ beforeBlock,
120
+ final: state,
121
+ mentionedBefore(blockIndex, name) {
122
+ const read = beforeBlock[blockIndex].sentencesRead;
123
+ return sentences.slice(0, read).some((sentence) => sentence.text.includes(name));
124
+ },
125
+ };
126
+ }