hanlint 0.0.7 → 0.0.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +32 -9
- package/data/adjectiveStems.txt +118 -0
- package/data/baselineAnchors.txt +3 -0
- package/data/blindEvaluation.schema.json +26 -0
- package/data/bridgeOpeners.txt +10 -0
- package/data/candidateActionNouns.txt +39 -0
- package/data/candidateSplitEndings.txt +13 -0
- package/data/cliches.json +1 -1
- package/data/confusable.json +2 -2
- package/data/deixis.txt +10 -3
- package/data/draftHistory.txt +10 -0
- package/data/easyWords.json +0 -42
- package/data/enoughMarkers.txt +6 -0
- package/data/entailmentPredictions.schema.json +52 -0
- package/data/euiNouns.txt +3 -4
- package/data/evidenceEntailmentBenchmark.schema.json +126 -0
- package/data/exemplars.json +323 -0
- package/data/holeKinds.json +2 -14
- package/data/iVerbStems.txt +26 -0
- package/data/inNouns.txt +147 -0
- package/data/irregularStems.txt +64 -0
- package/data/nonNouns.txt +153 -0
- package/data/panelAssignmentReview.schema.json +117 -0
- package/data/panelJudgePredictions.schema.json +78 -0
- package/data/panelReviewBatch.schema.json +79 -0
- package/data/panelTrialSet.schema.json +44 -0
- package/data/patterns.json +103 -0
- package/data/platformApis.txt +3 -1
- package/data/profiles.json +1 -0
- package/data/readerCalls.txt +12 -7
- package/data/redundantPair.json +1 -1
- package/data/rieulStems.txt +30 -0
- package/data/ruleCategories.json +52 -0
- package/data/ruleDocs.json +27 -19
- package/data/ruleMechanisms.json +52 -0
- package/data/spacing.json +0 -12
- package/data/spelling.json +7 -25
- package/data/translationese.json +10 -0
- package/data/verbTails.txt +17 -0
- package/data/version.json +1 -1
- package/data/writingBrief.schema.json +64 -0
- package/data/writingBriefV2.schema.json +116 -0
- package/data/writingTrial.schema.json +36 -0
- package/koglType1.LICENSE.md +25 -0
- package/package.json +18 -6
- package/src/analysis/grammar/ending.js +336 -0
- package/src/analysis/grammar/hangul.js +94 -0
- package/src/analysis/grammar/index.js +5 -0
- package/src/analysis/grammar/josa.js +73 -0
- package/src/analysis/grammar/register.js +115 -0
- package/src/analysis/grammar/voice.js +43 -0
- package/src/analysis/index.js +4 -31
- package/src/analysis/josa.js +2 -0
- package/src/analysis/{surface/splitSentences.js → splitSentences.js} +1 -1
- package/src/analysis/{surface/tokenize.js → tokenize.js} +70 -13
- package/src/baseline/store.js +114 -0
- package/src/cli/main.js +508 -61
- package/src/cli/walk.js +30 -0
- package/src/cli/welcome.js +52 -0
- package/src/config/settings.js +111 -12
- package/src/data/exemplars.js +177 -0
- package/src/data/load.js +10 -0
- package/src/data/operations.js +278 -0
- package/src/data/patches.js +125 -0
- package/src/data/patterns.js +48 -0
- package/src/data/profiles.js +102 -0
- package/src/document/model.js +2 -0
- package/src/document/parseMarkdown.js +84 -11
- package/src/edit/applyFixes.js +0 -0
- package/src/fingerprint/build.js +60 -39
- package/src/{rules/shared → fingerprint}/codeBlocks.js +11 -11
- package/src/fingerprint/codeMarkers.js +73 -0
- package/src/fingerprint/markers.js +9 -0
- package/src/fingerprint/readerState.js +126 -0
- package/src/fingerprint/sourceSentence.js +37 -0
- package/src/fingerprint/topics.js +1 -1
- package/src/index.js +2 -16
- package/src/report/compactReport.js +6 -1
- package/src/report/fingerprintJson.js +8 -3
- package/src/report/jsonReport.js +50 -3
- package/src/report/operationMatch.js +31 -0
- package/src/report/patchMatch.js +44 -0
- package/src/report/registerMatch.js +40 -0
- package/src/report/textReport.js +41 -5
- package/src/rules/code/duplicateBlock.js +4 -3
- package/src/rules/code/firstResultDistance.js +1 -0
- package/src/rules/code/inputFileSource.js +23 -68
- package/src/rules/code/installImport.js +6 -5
- package/src/rules/code/platformApi.js +7 -6
- package/src/rules/document/countMismatch.js +22 -6
- package/src/rules/document/enoughOnce.js +25 -0
- package/src/rules/document/fieldEcho.js +38 -0
- package/src/rules/document/keywordHeading.js +32 -0
- package/src/rules/document/keywordMissing.js +1 -0
- package/src/rules/document/noQuestion.js +5 -1
- package/src/rules/document/promiseRecall.js +4 -2
- package/src/rules/document/tableOddCell.js +61 -0
- package/src/rules/finding.js +24 -2
- package/src/rules/orthography/confusable.js +1 -0
- package/src/rules/orthography/spacing.js +1 -0
- package/src/rules/orthography/spelling.js +1 -0
- package/src/rules/paragraph/factListParagraph.js +1 -0
- package/src/rules/paragraph/paraFragment.js +21 -11
- package/src/rules/registry.js +70 -9
- package/src/rules/sentence/cliche.js +1 -0
- package/src/rules/sentence/connectorRepeat.js +8 -7
- package/src/rules/sentence/danglingDeixis.js +23 -3
- package/src/rules/sentence/dash.js +2 -1
- package/src/rules/sentence/deixis.js +7 -3
- package/src/rules/sentence/doubleNegative.js +4 -1
- package/src/rules/sentence/doublePassive.js +38 -1
- package/src/rules/sentence/draftHistory.js +24 -0
- package/src/rules/sentence/endingRepeat.js +6 -2
- package/src/rules/sentence/euiChain.js +3 -2
- package/src/rules/sentence/fillerOpener.js +1 -0
- package/src/rules/sentence/hardWord.js +1 -0
- package/src/rules/sentence/imperativePeriod.js +2 -1
- package/src/rules/sentence/japaneseLoan.js +1 -0
- package/src/rules/sentence/longSentence.js +3 -1
- package/src/rules/sentence/negationRedefine.js +12 -2
- package/src/rules/sentence/nounPile.js +1 -0
- package/src/rules/sentence/numberOrphan.js +45 -0
- package/src/rules/sentence/outsideProfile.js +68 -0
- package/src/rules/sentence/redundantPair.js +1 -0
- package/src/rules/sentence/translationese.js +1 -0
- package/src/rules/shared/candidates.js +129 -0
- package/src/rules/shared/dictionaryRule.js +12 -2
- package/src/rules/shared/localAntecedent.js +12 -0
- package/src/rules/shared/repeat.js +48 -0
- package/src/rules/structure/blockUnread.js +33 -0
- package/src/rules/structure/bridgeRepeat.js +40 -0
- package/src/rules/structure/emojiBullet.js +1 -0
- package/src/rules/structure/headingSentence.js +9 -4
- package/src/rules/structure/headingSkip.js +1 -0
- package/src/rules/structure/headingUniform.js +12 -20
- package/src/rules/structure/introImage.js +19 -0
- package/src/rules/structure/introLong.js +4 -1
- package/src/rules/structure/loneSubheading.js +42 -0
- package/src/rules/structure/moreLater.js +48 -0
- package/src/rules/structure/sectionNoProse.js +1 -0
- package/src/text.js +19 -0
- package/data/coverageTypes.txt +0 -7
- package/src/rules/document/readerAbsent.js +0 -11
- package/src/rules/paragraph/topicBreak.js +0 -25
- package/src/rules/sentence/ellipsis.js +0 -18
- package/src/rules/shared/runs.js +0 -22
- package/src/rules/structure/sectionResult.js +0 -37
|
@@ -4,15 +4,20 @@
|
|
|
4
4
|
* 빈 줄로 블록을 나누고 첫 줄로 종류를 정한다. 코드 펜스 안은 통째로 code 다. H2 가 절을 열고 첫 H2 앞은 도입 절이다.
|
|
5
5
|
*/
|
|
6
6
|
import { splitLines } from "../text.js";
|
|
7
|
-
import { CODE, EMBED, HEADING, HTML, IMAGE, LIST, PROSE, TABLE } from "./model.js";
|
|
7
|
+
import { CODE, EMBED, HEADING, HTML, IMAGE, LIST, PROSE, QUOTE, TABLE } from "./model.js";
|
|
8
8
|
|
|
9
9
|
const FRONTMATTER = /^---\r?\n([\s\S]*?)\r?\n---\r?\n/;
|
|
10
|
-
const
|
|
10
|
+
const FENCE_OPEN = /^\s*(?:(`{3,})([^`]*)|(~{3,})([^~]*))$/;
|
|
11
|
+
const FENCE_CLOSE = /^\s*(`{3,}|~{3,})\s*$/;
|
|
11
12
|
const HEADING_LINE = /^(#{1,6})\s+(.*?)\s*#*\s*$/;
|
|
13
|
+
const INDENTED_CODE_LINE = /^(?: {4}|\t)/;
|
|
14
|
+
const QUOTE_LINE = /^\s*>/;
|
|
12
15
|
const IMAGE_LINE = /^!\[/;
|
|
13
|
-
const LIST_LINE =
|
|
16
|
+
const LIST_LINE = /^(\s*)(?:[-*+]|\d+[.)])\s+/;
|
|
14
17
|
const TABLE_LINE = /^\s*\|/;
|
|
15
18
|
const URL_LINE = /^\s*https?:\/\/\S+\s*$/;
|
|
19
|
+
// 링크 하나만 있는 줄은 끼워 넣은 것 (영상 카드, 실행 칸, 참고 링크) 이라 주소만 있는 줄과 같은 embed 다. 파이썬과 같다.
|
|
20
|
+
const LINK_ONLY_LINE = /^\s*\[[^\]\n]+\]\(\S+?(?:\s+"[^"\n]*")?\)\s*$/;
|
|
16
21
|
const HTML_LINE = /^\s*</;
|
|
17
22
|
const META_KEY = /^[A-Za-z][A-Za-z0-9_]*$/;
|
|
18
23
|
const CONTROL = /^\s*<!--\s*hanlint-(disable-next-line|disable-next|disable|enable)\b([^>]*?)-->\s*$/;
|
|
@@ -34,16 +39,40 @@ export function parseFrontmatter(text) {
|
|
|
34
39
|
}
|
|
35
40
|
|
|
36
41
|
/** @param {string} firstLine */
|
|
37
|
-
export function classify(firstLine) {
|
|
42
|
+
export function classify(firstLine, lineCount = 1) {
|
|
43
|
+
if (INDENTED_CODE_LINE.test(firstLine)) return CODE;
|
|
38
44
|
if (HEADING_LINE.test(firstLine)) return HEADING;
|
|
45
|
+
if (QUOTE_LINE.test(firstLine)) return QUOTE;
|
|
39
46
|
if (IMAGE_LINE.test(firstLine)) return IMAGE;
|
|
40
47
|
if (TABLE_LINE.test(firstLine)) return TABLE;
|
|
41
48
|
if (LIST_LINE.test(firstLine)) return LIST;
|
|
42
|
-
if (URL_LINE.test(firstLine)) return EMBED;
|
|
49
|
+
if (URL_LINE.test(firstLine) || (lineCount === 1 && LINK_ONLY_LINE.test(firstLine))) return EMBED;
|
|
43
50
|
if (HTML_LINE.test(firstLine)) return HTML;
|
|
44
51
|
return PROSE;
|
|
45
52
|
}
|
|
46
53
|
|
|
54
|
+
/** @param {string} line */
|
|
55
|
+
function indentWidth(line) {
|
|
56
|
+
let width = 0;
|
|
57
|
+
for (const char of line) {
|
|
58
|
+
if (char === " ") width += 1;
|
|
59
|
+
else if (char === "\t") width += 4 - (width % 4);
|
|
60
|
+
else break;
|
|
61
|
+
}
|
|
62
|
+
return width;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
/** @param {string} line @param {number} width */
|
|
66
|
+
function afterIndent(line, width) {
|
|
67
|
+
let index = 0;
|
|
68
|
+
let consumed = 0;
|
|
69
|
+
while (index < line.length && consumed < width && (line[index] === " " || line[index] === "\t")) {
|
|
70
|
+
consumed += line[index] === " " ? 1 : 4 - (consumed % 4);
|
|
71
|
+
index += 1;
|
|
72
|
+
}
|
|
73
|
+
return line.slice(index);
|
|
74
|
+
}
|
|
75
|
+
|
|
47
76
|
/**
|
|
48
77
|
* @param {string} text
|
|
49
78
|
* @param {number} firstLine
|
|
@@ -61,10 +90,18 @@ export function splitBlocks(text, firstLine) {
|
|
|
61
90
|
let fenceStart = 0;
|
|
62
91
|
/** @type {string[]} */
|
|
63
92
|
let fenceLines = [];
|
|
93
|
+
/** @type {number | null} */
|
|
94
|
+
let listContinuation = null;
|
|
95
|
+
/** @type {number | null} */
|
|
96
|
+
let bufferListContinuation = null;
|
|
64
97
|
|
|
65
98
|
const flush = () => {
|
|
66
99
|
if (!buffer.length) return;
|
|
67
|
-
|
|
100
|
+
let firstLine = buffer[0];
|
|
101
|
+
if (bufferListContinuation !== null && indentWidth(firstLine) >= bufferListContinuation) {
|
|
102
|
+
firstLine = afterIndent(firstLine, bufferListContinuation);
|
|
103
|
+
}
|
|
104
|
+
const kind = classify(firstLine, buffer.length);
|
|
68
105
|
let joined = buffer.join("\n");
|
|
69
106
|
let level = 0;
|
|
70
107
|
if (kind === HEADING) {
|
|
@@ -74,6 +111,7 @@ export function splitBlocks(text, firstLine) {
|
|
|
74
111
|
}
|
|
75
112
|
blocks.push({ kind, startLine: bufferStart, endLine: bufferStart + buffer.length - 1, text: joined, level, index: blocks.length });
|
|
76
113
|
buffer = [];
|
|
114
|
+
bufferListContinuation = null;
|
|
77
115
|
};
|
|
78
116
|
|
|
79
117
|
for (const raw of splitLines(text)) {
|
|
@@ -81,17 +119,18 @@ export function splitBlocks(text, firstLine) {
|
|
|
81
119
|
const line = raw.replace(/\r+$/, "");
|
|
82
120
|
if (fence) {
|
|
83
121
|
fenceLines.push(line);
|
|
84
|
-
|
|
122
|
+
const closing = FENCE_CLOSE.exec(line);
|
|
123
|
+
if (closing && closing[1].startsWith(fence) && closing[1][0] === fence[0]) {
|
|
85
124
|
blocks.push({ kind: CODE, startLine: fenceStart, endLine: lineNo, text: fenceLines.join("\n"), level: 0, index: blocks.length });
|
|
86
125
|
fence = null;
|
|
87
126
|
fenceLines = [];
|
|
88
127
|
}
|
|
89
128
|
continue;
|
|
90
129
|
}
|
|
91
|
-
const opening =
|
|
130
|
+
const opening = FENCE_OPEN.exec(line);
|
|
92
131
|
if (opening) {
|
|
93
132
|
flush();
|
|
94
|
-
fence = opening[1];
|
|
133
|
+
fence = opening[1] || opening[3];
|
|
95
134
|
fenceStart = lineNo;
|
|
96
135
|
fenceLines = [line];
|
|
97
136
|
continue;
|
|
@@ -106,7 +145,17 @@ export function splitBlocks(text, firstLine) {
|
|
|
106
145
|
continue;
|
|
107
146
|
}
|
|
108
147
|
if (HEADING_LINE.test(line) && buffer.length) flush();
|
|
109
|
-
|
|
148
|
+
const listMatch = LIST_LINE.exec(line);
|
|
149
|
+
if (listMatch) {
|
|
150
|
+
const contentColumn = listMatch[0].length;
|
|
151
|
+
listContinuation = Math.ceil(contentColumn / 4) * 4;
|
|
152
|
+
} else if (listContinuation !== null && indentWidth(line) < listContinuation) {
|
|
153
|
+
listContinuation = null;
|
|
154
|
+
}
|
|
155
|
+
if (!buffer.length) {
|
|
156
|
+
bufferStart = lineNo;
|
|
157
|
+
bufferListContinuation = listContinuation;
|
|
158
|
+
}
|
|
110
159
|
buffer.push(line);
|
|
111
160
|
if (HEADING_LINE.test(line)) flush();
|
|
112
161
|
}
|
|
@@ -170,6 +219,30 @@ export function disabledRanges(blocks) {
|
|
|
170
219
|
return ranges;
|
|
171
220
|
}
|
|
172
221
|
|
|
222
|
+
/** 코드 블록 첫 줄의 언어 표기 (```python 의 python). 없으면 빈 문자열. 소문자로 준다. @param {string} blockText */
|
|
223
|
+
export function fenceLanguage(blockText) {
|
|
224
|
+
const opening = FENCE_OPEN.exec(blockText.split("\n", 1)[0]);
|
|
225
|
+
if (!opening) return "";
|
|
226
|
+
const info = (opening[2] || opening[4] || "").trim();
|
|
227
|
+
return info ? info.split(/\s+/)[0].toLowerCase() : "";
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
/**
|
|
231
|
+
* 언어 표기가 languages 에 든 펜스를 없앤 문서. 블록 순서 번호를 다시 매기고 절을 다시 묶는다. 인라인 제어는 원문 줄
|
|
232
|
+
* 기준이라 그대로 두고, 뺀 펜스는 ignored 에 남긴다. 파이썬 dropFences 와 같다.
|
|
233
|
+
* @param {import("./model.js").Document} doc
|
|
234
|
+
* @param {string[]} languages
|
|
235
|
+
* @returns {import("./model.js").Document}
|
|
236
|
+
*/
|
|
237
|
+
export function dropFences(doc, languages) {
|
|
238
|
+
const wanted = new Set(languages.map((name) => name.trim().toLowerCase()).filter(Boolean));
|
|
239
|
+
if (!wanted.size) return doc;
|
|
240
|
+
const dropped = doc.blocks.filter((block) => block.kind === CODE && wanted.has(fenceLanguage(block.text)));
|
|
241
|
+
if (!dropped.length) return doc;
|
|
242
|
+
const blocks = doc.blocks.filter((block) => !dropped.includes(block)).map((block, index) => ({ ...block, index }));
|
|
243
|
+
return { path: doc.path, frontmatter: doc.frontmatter, blocks, sections: groupSections(blocks), disabled: doc.disabled, ignored: [...doc.ignored, ...dropped] };
|
|
244
|
+
}
|
|
245
|
+
|
|
173
246
|
/**
|
|
174
247
|
* @param {string} text
|
|
175
248
|
* @param {string | null} [path]
|
|
@@ -179,5 +252,5 @@ export function parseMarkdown(text, path = null) {
|
|
|
179
252
|
const { meta, firstLine } = parseFrontmatter(text);
|
|
180
253
|
const body = firstLine === 1 ? text : splitLines(text).slice(firstLine - 1).join("\n");
|
|
181
254
|
const blocks = splitBlocks(body, firstLine);
|
|
182
|
-
return { path, frontmatter: meta, blocks, sections: groupSections(blocks), disabled: disabledRanges(blocks) };
|
|
255
|
+
return { path, frontmatter: meta, blocks, sections: groupSections(blocks), disabled: disabledRanges(blocks), ignored: [] };
|
|
183
256
|
}
|
package/src/edit/applyFixes.js
CHANGED
|
Binary file
|
package/src/fingerprint/build.js
CHANGED
|
@@ -1,11 +1,16 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
/** 문서 모델을 한 번 훑어 글 지문을 만든다. 텍스트를 읽는 유일한 자리다. 파이썬 fingerprint/build.py 와 같다. */
|
|
3
3
|
import { defaultConfig } from "../config/settings.js";
|
|
4
|
+
import { NONE as NO_REGISTER, documentRegister, lastWord, registerOfWord } from "../analysis/grammar/register.js";
|
|
5
|
+
import { doublePassiveSpans, euiCount, longestNounRun, splitSentences } from "../analysis/index.js";
|
|
4
6
|
import { HEADING, PROSE, sectionStartLine, sectionTitle } from "../document/model.js";
|
|
7
|
+
import { dropFences } from "../document/parseMarkdown.js";
|
|
5
8
|
import { codeSpans, plainText } from "../document/plainText.js";
|
|
6
9
|
import { countIn, mean, pstdev, wordCount } from "../text.js";
|
|
10
|
+
import { codeBlocksOf } from "./codeBlocks.js";
|
|
7
11
|
import { entriesFor, matchesIn } from "./dictionaries.js";
|
|
8
12
|
import * as markers from "./markers.js";
|
|
13
|
+
import { buildReaderTrail } from "./readerState.js";
|
|
9
14
|
import { overlap, topicsOf, unionOf } from "./topics.js";
|
|
10
15
|
|
|
11
16
|
/**
|
|
@@ -19,6 +24,7 @@ import { overlap, topicsOf, unionOf } from "./topics.js";
|
|
|
19
24
|
* @property {number} length 어절 수
|
|
20
25
|
* @property {string} ending
|
|
21
26
|
* @property {string} mood
|
|
27
|
+
* @property {string} register
|
|
22
28
|
* @property {number} commas
|
|
23
29
|
* @property {string | null} connectorStart
|
|
24
30
|
* @property {number} causal
|
|
@@ -81,28 +87,31 @@ import { overlap, topicsOf, unionOf } from "./topics.js";
|
|
|
81
87
|
* @property {number} questionCount
|
|
82
88
|
* @property {number} readerCallCount
|
|
83
89
|
* @property {[number, string, number, string][]} countPromises (수, 단위, 줄, 원문)
|
|
84
|
-
* @property {[
|
|
85
|
-
* @property {
|
|
90
|
+
* @property {import("./codeBlocks.js").CodeBlock[]} codeBlocks 코드 펜스를 언어와 본문 줄로 푼 것
|
|
91
|
+
* @property {import("./readerState.js").ReaderTrail} reader 자리마다의 독자 상태. 약속과 회수 표지는 final 에 있다
|
|
86
92
|
* @property {[string, number, number][]} disabled
|
|
93
|
+
* @property {import("../document/model.js").Block[]} ignored 설정이 지문에서 뺀 펜스
|
|
94
|
+
* @property {string} register
|
|
95
|
+
* @property {number} registerShare
|
|
87
96
|
* @property {SectionPrint} intro
|
|
88
97
|
* @property {SectionPrint[]} bodySections
|
|
89
98
|
*/
|
|
90
99
|
|
|
91
100
|
/**
|
|
92
|
-
* 문장 안의 인용 구간.
|
|
93
|
-
*
|
|
94
|
-
* @param {
|
|
101
|
+
* 문장 안의 인용 구간. 블록 좌표의 구간을 문장 좌표로 옮긴다. 따옴표 쌍도 블록에서 재서 넘겨받는다.
|
|
102
|
+
* 문장 안에서 재면 문장 분리가 따옴표 안에서 잘릴 때 쌍이 안 잡혀 인용 예외가 풀린다. 파이썬과 같다.
|
|
103
|
+
* @param {import("../analysis/splitSentences.js").Sentence} sentence
|
|
104
|
+
* @param {[number, number][]} blockSpans
|
|
95
105
|
* @returns {[number, number][]}
|
|
96
106
|
*/
|
|
97
|
-
function quotedIn(sentence,
|
|
107
|
+
function quotedIn(sentence, blockSpans) {
|
|
98
108
|
/** @type {[number, number][]} */
|
|
99
109
|
const spans = [];
|
|
100
|
-
for (const [start, end] of
|
|
110
|
+
for (const [start, end] of blockSpans) {
|
|
101
111
|
const lo = Math.max(start, sentence.start);
|
|
102
112
|
const hi = Math.min(end, sentence.end);
|
|
103
113
|
if (lo < hi) spans.push([lo - sentence.start, hi - sentence.start]);
|
|
104
114
|
}
|
|
105
|
-
spans.push(...markers.quoteSpans(sentence.text));
|
|
106
115
|
const unique = new Map(spans.map((span) => [`${span[0]},${span[1]}`, span]));
|
|
107
116
|
return [...unique.values()].sort((a, b) => a[0] - b[0] || a[1] - b[1]);
|
|
108
117
|
}
|
|
@@ -114,35 +123,39 @@ function quotedIn(sentence, blockCodeSpans) {
|
|
|
114
123
|
* @param {import("../document/model.js").Block} block
|
|
115
124
|
* @param {number} paragraphIndex
|
|
116
125
|
* @param {number} sectionIndex
|
|
117
|
-
* @param {import("../analysis/index.js").Analyzer} analyzer
|
|
118
126
|
* @param {import("./dictionaries.js").Entry[]} entries
|
|
119
127
|
* @param {[number, number][]} quoted
|
|
120
128
|
* @returns {SentencePrint}
|
|
121
129
|
*/
|
|
122
|
-
function makeSentencePrint(
|
|
130
|
+
function makeSentencePrint(build, text, line, block, sectionIndex, quoted) {
|
|
131
|
+
const { entries } = build;
|
|
123
132
|
const ending = markers.endingOf(text);
|
|
124
133
|
const deixis = markers
|
|
125
134
|
.matchedSpans(text, "deixis.txt")
|
|
126
135
|
.filter(([start, end]) => !markers.insideAny(start, end, quoted))
|
|
127
136
|
.map(([, , found]) => found);
|
|
128
137
|
const matches = matchesIn(text, entries).filter((m) => !markers.insideAny(m.start, m.end, quoted));
|
|
138
|
+
const mood = markers.moodOf(text, ending);
|
|
129
139
|
return {
|
|
130
|
-
index,
|
|
140
|
+
index: build.sentences.length,
|
|
131
141
|
line,
|
|
132
142
|
text,
|
|
133
143
|
blockIndex: block.index,
|
|
134
|
-
paragraphIndex,
|
|
144
|
+
paragraphIndex: build.paragraphs.length,
|
|
135
145
|
sectionIndex,
|
|
136
146
|
length: wordCount(text),
|
|
137
147
|
ending,
|
|
138
|
-
mood
|
|
148
|
+
mood,
|
|
149
|
+
register: mood === "평서" ? (registerOfWord(lastWord(text)) ?? NO_REGISTER) : NO_REGISTER,
|
|
139
150
|
commas: markers.countCommas(text),
|
|
140
151
|
connectorStart: markers.connectorStartOf(text),
|
|
141
152
|
causal: markers.countMatches(text, "causalMarkers.txt"),
|
|
142
153
|
deixis,
|
|
143
|
-
euiCount:
|
|
144
|
-
nounRun:
|
|
145
|
-
passives:
|
|
154
|
+
euiCount: euiCount(text),
|
|
155
|
+
nounRun: longestNounRun(text),
|
|
156
|
+
passives: doublePassiveSpans(text)
|
|
157
|
+
.filter(([start, end]) => !markers.insideAny(start, end, quoted))
|
|
158
|
+
.map(([, , surface]) => surface),
|
|
146
159
|
hedges: markers.countMatches(text, "hedges.txt"),
|
|
147
160
|
numbers: markers.countNumbers(text),
|
|
148
161
|
topics: topicsOf(text),
|
|
@@ -164,11 +177,11 @@ function makeSentencePrint(text, index, line, block, paragraphIndex, sectionInde
|
|
|
164
177
|
* @param {boolean} followsProseDirectly
|
|
165
178
|
* @returns {ParagraphPrint}
|
|
166
179
|
*/
|
|
167
|
-
function makeParagraphPrint(
|
|
180
|
+
function makeParagraphPrint(build, sentences, block, sectionIndex, previous, followsProseDirectly) {
|
|
168
181
|
const lengths = sentences.map((s) => s.length);
|
|
169
182
|
const topics = unionOf(sentences.map((s) => s.topics));
|
|
170
183
|
return {
|
|
171
|
-
index,
|
|
184
|
+
index: build.paragraphs.length,
|
|
172
185
|
blockIndex: block.index,
|
|
173
186
|
sectionIndex,
|
|
174
187
|
startLine: block.startLine,
|
|
@@ -189,13 +202,13 @@ function makeParagraphPrint(sentences, index, block, sectionIndex, previous, fol
|
|
|
189
202
|
/**
|
|
190
203
|
* @param {import("../document/model.js").Section} section
|
|
191
204
|
* @param {number} sectionIndex
|
|
192
|
-
* @param {import("../analysis/index.js").Analyzer} analyzer
|
|
193
205
|
* @param {import("./dictionaries.js").Entry[]} entries
|
|
194
206
|
* @param {SentencePrint[]} sentences
|
|
195
207
|
* @param {ParagraphPrint[]} paragraphs
|
|
196
208
|
* @returns {SectionPrint}
|
|
197
209
|
*/
|
|
198
|
-
function buildSection(section, sectionIndex
|
|
210
|
+
function buildSection(build, section, sectionIndex) {
|
|
211
|
+
const { sentences, paragraphs } = build;
|
|
199
212
|
/** @type {ParagraphPrint[]} */
|
|
200
213
|
const sectionParagraphs = [];
|
|
201
214
|
/** @type {import("../document/model.js").Block | null} */
|
|
@@ -206,22 +219,24 @@ function buildSection(section, sectionIndex, analyzer, entries, sentences, parag
|
|
|
206
219
|
continue;
|
|
207
220
|
}
|
|
208
221
|
const text = plainText(block.text);
|
|
209
|
-
|
|
222
|
+
// 인라인 코드와 따옴표 쌍을 둘 다 블록 좌표에서 잰다. 문장 분리가 인용 안에서 잘려도 쌍이 남는다.
|
|
223
|
+
const blockSpans = [...codeSpans(block.text, text), ...markers.quoteSpans(text)].sort(
|
|
224
|
+
(a, b) => a[0] - b[0] || a[1] - b[1],
|
|
225
|
+
);
|
|
210
226
|
/** @type {SentencePrint[]} */
|
|
211
227
|
const blockSentences = [];
|
|
212
|
-
for (const sentence of
|
|
228
|
+
for (const sentence of splitSentences(text)) {
|
|
213
229
|
const line = block.startLine + countIn(text, "\n", sentence.start);
|
|
214
|
-
const
|
|
215
|
-
const
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
);
|
|
230
|
+
const quoted = quotedIn(sentence, blockSpans);
|
|
231
|
+
const made = makeSentencePrint(build, sentence.text, line, block, sectionIndex, quoted);
|
|
232
|
+
// 만든 자리에서 바로 쌓는다. 다음 문장의 index 가 이 길이에서 나오므로 미루면 셈이 어긋난다.
|
|
233
|
+
sentences.push(made);
|
|
234
|
+
blockSentences.push(made);
|
|
219
235
|
}
|
|
220
|
-
sentences.push(...blockSentences);
|
|
221
236
|
const previous = sectionParagraphs.length ? sectionParagraphs[sectionParagraphs.length - 1] : null;
|
|
222
237
|
const paragraph = makeParagraphPrint(
|
|
238
|
+
build,
|
|
223
239
|
blockSentences,
|
|
224
|
-
paragraphs.length,
|
|
225
240
|
block,
|
|
226
241
|
sectionIndex,
|
|
227
242
|
previous,
|
|
@@ -246,23 +261,26 @@ function buildSection(section, sectionIndex, analyzer, entries, sentences, parag
|
|
|
246
261
|
|
|
247
262
|
/**
|
|
248
263
|
* @param {import("../document/model.js").Document} doc
|
|
249
|
-
* @param {import("../analysis/index.js").Analyzer} analyzer
|
|
250
264
|
* @param {import("../config/settings.js").Config} [config]
|
|
251
265
|
* @returns {DocumentPrint}
|
|
252
266
|
*/
|
|
253
|
-
export function buildFingerprint(doc,
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
const
|
|
257
|
-
|
|
258
|
-
const paragraphs = [];
|
|
267
|
+
export function buildFingerprint(doc, config = defaultConfig()) {
|
|
268
|
+
// 코드도 산문도 아닌 펜스는 지문에 들어오기 전에 뺀다. 모든 진입점이 여기를 지나므로 한 자리면 된다.
|
|
269
|
+
doc = dropFences(doc, config.ignoreFences);
|
|
270
|
+
const build = { entries: entriesFor(config), sentences: /** @type {SentencePrint[]} */ ([]), paragraphs: /** @type {ParagraphPrint[]} */ ([]) };
|
|
271
|
+
const { sentences, paragraphs } = build;
|
|
259
272
|
const sections = doc.sections.map((section, index) =>
|
|
260
|
-
buildSection(section, index
|
|
273
|
+
buildSection(build, section, index),
|
|
261
274
|
);
|
|
262
275
|
const headingBlocks = doc.blocks.filter((b) => b.kind === HEADING);
|
|
263
276
|
/** @type {[number, string, number][]} */
|
|
264
277
|
const headings = headingBlocks.map((b) => [b.level, b.text, b.startLine]);
|
|
265
278
|
const headingQuestions = headingBlocks.filter((b) => b.text.includes("?")).length;
|
|
279
|
+
const [register, registerShare] = documentRegister(
|
|
280
|
+
sentences.filter((sentence) => sentence.mood === "평서").map((sentence) => lastWord(sentence.text)),
|
|
281
|
+
config.registerMinShare,
|
|
282
|
+
);
|
|
283
|
+
const codeBlocks = codeBlocksOf(doc.blocks);
|
|
266
284
|
return {
|
|
267
285
|
path: doc.path,
|
|
268
286
|
frontmatter: { ...doc.frontmatter },
|
|
@@ -275,9 +293,12 @@ export function buildFingerprint(doc, analyzer, config = defaultConfig()) {
|
|
|
275
293
|
questionCount: sentences.filter((s) => s.mood === "의문").length + headingQuestions,
|
|
276
294
|
readerCallCount: sentences.filter((s) => s.readerCall || s.mood === "명령").length,
|
|
277
295
|
countPromises: sentences.flatMap((s) => s.countPromises.map(([n, unit, text]) => [n, unit, s.line, text])),
|
|
278
|
-
|
|
279
|
-
|
|
296
|
+
codeBlocks,
|
|
297
|
+
reader: buildReaderTrail(doc.blocks, codeBlocks, sentences),
|
|
298
|
+
register,
|
|
299
|
+
registerShare,
|
|
280
300
|
disabled: doc.disabled,
|
|
301
|
+
ignored: doc.ignored,
|
|
281
302
|
intro: sections[0],
|
|
282
303
|
bodySections: sections.filter((s) => !s.isIntro),
|
|
283
304
|
};
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
// @ts-check
|
|
2
2
|
/**
|
|
3
|
-
* 코드 블록을 언어와 본문으로 나눈다.
|
|
3
|
+
* 코드 블록을 언어와 본문으로 나눈다. 지문이 한 번 만들고 (DocumentPrint.codeBlocks) code 부류 규칙과 독자 상태가
|
|
4
|
+
* 읽는다. 파이썬 fingerprint/codeBlocks.py 와 같다.
|
|
4
5
|
* 펜스 첫 줄의 언어 표기 (```python) 를 읽고 본문 줄에 원문 줄 번호를 붙인다. text 펜스는 출력이다.
|
|
5
6
|
*/
|
|
6
|
-
import { CODE } from "
|
|
7
|
+
import { CODE } from "../document/model.js";
|
|
8
|
+
import { fenceLanguage } from "../document/parseMarkdown.js";
|
|
7
9
|
|
|
8
|
-
const FENCE_LANGUAGE = /^\s*(?:```|~~~)\s*([A-Za-z0-9_+-]*)/;
|
|
9
10
|
const CLOSING_FENCE = /^\s*(?:```|~~~)\s*$/;
|
|
10
11
|
|
|
11
12
|
/**
|
|
@@ -18,20 +19,19 @@ const CLOSING_FENCE = /^\s*(?:```|~~~)\s*$/;
|
|
|
18
19
|
* @property {boolean} isOutput
|
|
19
20
|
*/
|
|
20
21
|
|
|
21
|
-
/** @param {import("
|
|
22
|
-
export function codeBlocksOf(
|
|
22
|
+
/** @param {import("../document/model.js").Block[]} blocks @returns {CodeBlock[]} */
|
|
23
|
+
export function codeBlocksOf(blocks) {
|
|
23
24
|
/** @type {CodeBlock[]} */
|
|
24
|
-
const
|
|
25
|
-
for (const block of
|
|
25
|
+
const found = [];
|
|
26
|
+
for (const block of blocks) {
|
|
26
27
|
if (block.kind !== CODE) continue;
|
|
27
28
|
const raw = block.text.split("\n");
|
|
28
|
-
const
|
|
29
|
-
const language = (match ? match[1] : "").toLowerCase();
|
|
29
|
+
const language = fenceLanguage(block.text);
|
|
30
30
|
let body = raw.slice(1);
|
|
31
31
|
if (body.length && CLOSING_FENCE.test(body[body.length - 1])) body = body.slice(0, -1);
|
|
32
32
|
/** @type {[number, string][]} */
|
|
33
33
|
const lines = body.map((line, offset) => [block.startLine + 1 + offset, line]);
|
|
34
|
-
|
|
34
|
+
found.push({
|
|
35
35
|
index: block.index,
|
|
36
36
|
startLine: block.startLine,
|
|
37
37
|
language,
|
|
@@ -40,5 +40,5 @@ export function codeBlocksOf(doc) {
|
|
|
40
40
|
isOutput: ["text", "", "output", "console"].includes(language),
|
|
41
41
|
});
|
|
42
42
|
}
|
|
43
|
-
return
|
|
43
|
+
return found;
|
|
44
44
|
}
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// @ts-check
|
|
2
|
+
/**
|
|
3
|
+
* 코드 줄에서 파일의 공급과 수요를 읽는다. 어떤 줄이 파일을 만들고 (쓰기, 폴더 만들기) 어떤 줄이 파일을 읽는가.
|
|
4
|
+
* 파이썬 fingerprint/codeMarkers.py 와 같다. 줄마다 대조하므로 여러 줄에 걸친 호출은 보지 않는다.
|
|
5
|
+
*/
|
|
6
|
+
|
|
7
|
+
const QUOTED = "\\(\\s*[\"']([^\"'\\n]+)[\"']";
|
|
8
|
+
const READS = new RegExp(
|
|
9
|
+
"\\b(?:read_csv|read_excel|read_parquet|read_json|read_sql|read_table|read_ndjson|scan_csv|scan_parquet|scan_ndjson|" +
|
|
10
|
+
"read_text|read_bytes|loadtxt|imread|load_workbook)" + QUOTED,
|
|
11
|
+
"g",
|
|
12
|
+
);
|
|
13
|
+
const OPENS = new RegExp("\\bopen" + QUOTED + "\\s*(?:,\\s*(?:mode\\s*=\\s*)?[\"']([^\"']*)[\"'])?", "g");
|
|
14
|
+
const WRITES = new RegExp(
|
|
15
|
+
"\\b(?:to_csv|to_excel|to_parquet|to_json|to_sql|write_csv|write_parquet|write_json|write_ndjson|sink_parquet|sink_csv|" +
|
|
16
|
+
"write_text|write_bytes|savefig|save|imwrite|savetxt|dump)" + QUOTED,
|
|
17
|
+
"g",
|
|
18
|
+
);
|
|
19
|
+
const SHELL_WRITES = /(?:>>?|-o|-O|--output|Out-File)\s*["']?([\w./\\-]+\.[A-Za-z0-9]+)/g;
|
|
20
|
+
const SQL_PATHS = /FROM\s+'([^']+\.(?:csv|parquet|json|xlsx))'/gi;
|
|
21
|
+
const MAKES_DIR = /(?:mkdir(?:\s+-p)?|makedirs|md)\s*\(?\s*["']?([\w./\\-]+)|Path\(\s*["']([^"']+)["']\s*\)\.mkdir/g;
|
|
22
|
+
/** open 의 모드에 이 글자가 있으면 쓰기다. 없으면 (기본 r) 읽기다. */
|
|
23
|
+
const WRITE_MODES = [..."wax"];
|
|
24
|
+
|
|
25
|
+
/** @param {string} path */
|
|
26
|
+
export function fileName(path) {
|
|
27
|
+
const parts = path.replace(/\\/g, "/").replace(/\/+$/, "").split("/");
|
|
28
|
+
return parts[parts.length - 1];
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
/** @param {string | undefined} mode */
|
|
32
|
+
function isWriteMode(mode) {
|
|
33
|
+
return WRITE_MODES.some((flag) => (mode ?? "r").includes(flag));
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** @param {RegExp} pattern @param {string} text @param {number} group */
|
|
37
|
+
function all(pattern, text, group) {
|
|
38
|
+
const found = [];
|
|
39
|
+
for (const match of text.matchAll(pattern)) if (match[group] !== undefined) found.push(match[group]);
|
|
40
|
+
return found;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* 코드 줄들이 만드는 [파일 이름, 폴더 이름]. 쓰기 함수, 셸 리다이렉션, 쓰기 모드의 open, mkdir 을 본다.
|
|
45
|
+
* @param {string[]} lines
|
|
46
|
+
* @returns {[Set<string>, Set<string>]}
|
|
47
|
+
*/
|
|
48
|
+
export function createdIn(lines) {
|
|
49
|
+
/** @type {Set<string>} */
|
|
50
|
+
const files = new Set();
|
|
51
|
+
/** @type {Set<string>} */
|
|
52
|
+
const dirs = new Set();
|
|
53
|
+
for (const code of lines) {
|
|
54
|
+
for (const path of all(WRITES, code, 1)) files.add(fileName(path));
|
|
55
|
+
for (const path of all(SHELL_WRITES, code, 1)) files.add(fileName(path));
|
|
56
|
+
for (const match of code.matchAll(OPENS)) if (isWriteMode(match[2])) files.add(fileName(match[1]));
|
|
57
|
+
for (const match of code.matchAll(MAKES_DIR)) dirs.add((match[1] ?? match[2]).replace(/\\/g, "/").split("/")[0]);
|
|
58
|
+
}
|
|
59
|
+
return [files, dirs];
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
/** 한 줄이 읽는 경로. 읽기 함수, 읽기 모드의 open, SQL 의 FROM 차례다. @param {string} code */
|
|
63
|
+
export function readsIn(code) {
|
|
64
|
+
const reads = all(READS, code, 1);
|
|
65
|
+
for (const match of code.matchAll(OPENS)) if (!isWriteMode(match[2])) reads.push(match[1]);
|
|
66
|
+
reads.push(...all(SQL_PATHS, code, 1));
|
|
67
|
+
return reads;
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
/** 한 줄이 쓰기 함수로 쓰는 경로 그대로. 그 경로의 폴더가 있는지 볼 때 쓴다. @param {string} code */
|
|
71
|
+
export function writeTargets(code) {
|
|
72
|
+
return all(WRITES, code, 1);
|
|
73
|
+
}
|
|
@@ -8,6 +8,8 @@ import { compile, escape } from "../regex.js";
|
|
|
8
8
|
|
|
9
9
|
const TRAILING = /[\s.?!"'”’)\]]+$/;
|
|
10
10
|
const NUMBER = /\d+(?:[.,]\d+)*/g;
|
|
11
|
+
/** 값으로 견주는 수. NUMBER 는 수의 개수를 세고 이것은 수의 정체를 센다. 천 단위 쉼표까지 한 수로 읽는다. */
|
|
12
|
+
const NUMERAL = /\d[\d,]*(?:\.\d+)?/g;
|
|
11
13
|
const COMMA = /,/g;
|
|
12
14
|
// 따옴표 쌍. 안은 사용이 아니라 인용이다. 작은따옴표는 앞이 문장 첫머리나 공백이나 여는 괄호일 때만.
|
|
13
15
|
const QUOTE_PAIRS = [
|
|
@@ -165,6 +167,13 @@ export function countPromisesIn(text) {
|
|
|
165
167
|
}
|
|
166
168
|
|
|
167
169
|
/** @param {string} text */
|
|
170
|
+
/** 글에 나온 수의 정체 (천 단위 쉼표를 뗀 꼴). 독자 상태가 쌓고 numberOrphan 이 견준다. @param {string} text @returns {Set<string>} */
|
|
171
|
+
export function numeralsIn(text) {
|
|
172
|
+
const found = new Set();
|
|
173
|
+
for (const match of text.matchAll(NUMERAL)) found.add(match[0].replaceAll(",", ""));
|
|
174
|
+
return found;
|
|
175
|
+
}
|
|
176
|
+
|
|
168
177
|
export function countNumbers(text) {
|
|
169
178
|
return [...text.matchAll(NUMBER)].length;
|
|
170
179
|
}
|