@wdprlib/parser 5.0.0 → 5.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/index.cjs +250 -144
- package/dist/index.d.cts +8 -13
- package/dist/index.d.ts +8 -13
- package/dist/index.js +230 -124
- package/package.json +2 -2
- package/src/parser/parse/context.ts +7 -2
- package/src/parser/parse/index.ts +21 -3
- package/src/parser/parse/parser.ts +37 -28
- package/src/parser/rules/block/include/index.ts +20 -2
- package/src/parser/rules/block/module/include/resolve/index.ts +28 -0
- package/src/parser/rules/block/module/include/resolve/iterate.ts +28 -4
- package/src/parser/rules/block/module/include/scanner.ts +58 -1
- package/src/parser/rules/contracts/parse-context.ts +2 -0
- package/src/pipeline/process.ts +48 -31
package/dist/index.js
CHANGED
|
@@ -2,13 +2,13 @@
|
|
|
2
2
|
import {
|
|
3
3
|
createPoint,
|
|
4
4
|
createPosition,
|
|
5
|
-
text,
|
|
5
|
+
text as text2,
|
|
6
6
|
container,
|
|
7
|
-
paragraph,
|
|
7
|
+
paragraph as paragraph2,
|
|
8
8
|
bold,
|
|
9
9
|
italics,
|
|
10
10
|
heading,
|
|
11
|
-
lineBreak,
|
|
11
|
+
lineBreak as lineBreak2,
|
|
12
12
|
horizontalRule,
|
|
13
13
|
link,
|
|
14
14
|
list,
|
|
@@ -6590,13 +6590,40 @@ function scanIncludeDirectives(source) {
|
|
|
6590
6590
|
`, start);
|
|
6591
6591
|
let depth = 0;
|
|
6592
6592
|
let linkDepth = 0;
|
|
6593
|
+
const nestedQuotes = [];
|
|
6594
|
+
const nestedQuoteAllowed = [];
|
|
6593
6595
|
let i = start;
|
|
6594
6596
|
let closeEnd = -1;
|
|
6595
6597
|
while (i < source.length) {
|
|
6596
6598
|
const ch = source.charCodeAt(i);
|
|
6597
6599
|
const next = source.charCodeAt(i + 1);
|
|
6598
6600
|
const nextNext = source.charCodeAt(i + 2);
|
|
6599
|
-
if (ch ===
|
|
6601
|
+
if (ch === 10 || ch === 13) {
|
|
6602
|
+
linkDepth = 0;
|
|
6603
|
+
if (depth > 1) {
|
|
6604
|
+
depth = 1;
|
|
6605
|
+
nestedQuotes.length = 0;
|
|
6606
|
+
nestedQuoteAllowed.length = 0;
|
|
6607
|
+
}
|
|
6608
|
+
i++;
|
|
6609
|
+
continue;
|
|
6610
|
+
}
|
|
6611
|
+
const nestedQuote = nestedQuotes.at(-1);
|
|
6612
|
+
if (nestedQuote !== undefined && nestedQuote !== null) {
|
|
6613
|
+
if (ch === 92) {
|
|
6614
|
+
i += 2;
|
|
6615
|
+
} else {
|
|
6616
|
+
if (ch === nestedQuote) {
|
|
6617
|
+
nestedQuotes[nestedQuotes.length - 1] = null;
|
|
6618
|
+
nestedQuoteAllowed[nestedQuoteAllowed.length - 1] = false;
|
|
6619
|
+
}
|
|
6620
|
+
i++;
|
|
6621
|
+
}
|
|
6622
|
+
} else if (nestedQuote === null && nestedQuoteAllowed.at(-1) === true && (ch === 34 || ch === 39)) {
|
|
6623
|
+
nestedQuotes[nestedQuotes.length - 1] = ch;
|
|
6624
|
+
nestedQuoteAllowed[nestedQuoteAllowed.length - 1] = false;
|
|
6625
|
+
i++;
|
|
6626
|
+
} else if (ch === OPEN_BRACKET && next === OPEN_BRACKET && nextNext === OPEN_BRACKET) {
|
|
6600
6627
|
linkDepth++;
|
|
6601
6628
|
i += 3;
|
|
6602
6629
|
} else if (linkDepth > 0 && ch === CLOSE_BRACKET && next === CLOSE_BRACKET && nextNext === CLOSE_BRACKET) {
|
|
@@ -6605,10 +6632,26 @@ function scanIncludeDirectives(source) {
|
|
|
6605
6632
|
} else if (linkDepth > 0) {
|
|
6606
6633
|
i++;
|
|
6607
6634
|
} else if (ch === OPEN_BRACKET && next === OPEN_BRACKET) {
|
|
6635
|
+
if (nestedQuoteAllowed.length > 0) {
|
|
6636
|
+
nestedQuoteAllowed[nestedQuoteAllowed.length - 1] = false;
|
|
6637
|
+
}
|
|
6638
|
+
if (depth > 0) {
|
|
6639
|
+
nestedQuotes.push(null);
|
|
6640
|
+
nestedQuoteAllowed.push(false);
|
|
6641
|
+
}
|
|
6608
6642
|
depth++;
|
|
6609
6643
|
i += 2;
|
|
6644
|
+
} else if (ch === CLOSE_BRACKET && next !== CLOSE_BRACKET && depth > 1) {
|
|
6645
|
+
depth--;
|
|
6646
|
+
nestedQuotes.pop();
|
|
6647
|
+
nestedQuoteAllowed.pop();
|
|
6648
|
+
i++;
|
|
6610
6649
|
} else if (ch === CLOSE_BRACKET && next === CLOSE_BRACKET) {
|
|
6611
6650
|
const closeStart = i;
|
|
6651
|
+
if (depth > 1) {
|
|
6652
|
+
nestedQuotes.pop();
|
|
6653
|
+
nestedQuoteAllowed.pop();
|
|
6654
|
+
}
|
|
6612
6655
|
depth--;
|
|
6613
6656
|
i += 2;
|
|
6614
6657
|
if (depth <= 0) {
|
|
@@ -6625,6 +6668,13 @@ function scanIncludeDirectives(source) {
|
|
|
6625
6668
|
}
|
|
6626
6669
|
}
|
|
6627
6670
|
} else {
|
|
6671
|
+
if (nestedQuoteAllowed.length > 0) {
|
|
6672
|
+
if (ch === 61) {
|
|
6673
|
+
nestedQuoteAllowed[nestedQuoteAllowed.length - 1] = true;
|
|
6674
|
+
} else if (ch !== 32 && ch !== 9) {
|
|
6675
|
+
nestedQuoteAllowed[nestedQuoteAllowed.length - 1] = false;
|
|
6676
|
+
}
|
|
6677
|
+
}
|
|
6628
6678
|
i++;
|
|
6629
6679
|
}
|
|
6630
6680
|
}
|
|
@@ -6810,13 +6860,13 @@ function expandIterativeWithTrace(source, fetcher, maxIterations) {
|
|
|
6810
6860
|
reachedMaxIterations: iterations.length === maxIterations && scanIncludeDirectives(current2).length > 0
|
|
6811
6861
|
};
|
|
6812
6862
|
}
|
|
6813
|
-
async function expandIterativeAsyncWithTrace(source, fetcher, maxIterations) {
|
|
6863
|
+
async function expandIterativeAsyncWithTrace(source, fetcher, maxIterations, shouldDefer) {
|
|
6814
6864
|
let current2 = source;
|
|
6815
6865
|
const replacementCache = new Map;
|
|
6816
6866
|
const dependencies = [];
|
|
6817
6867
|
const iterations = [];
|
|
6818
6868
|
for (let i = 0;i < maxIterations; i++) {
|
|
6819
|
-
const expanded = await expandOneIterationAsyncWithTrace(current2, fetcher, replacementCache, i);
|
|
6869
|
+
const expanded = await expandOneIterationAsyncWithTrace(current2, fetcher, replacementCache, i, shouldDefer);
|
|
6820
6870
|
if (expanded === null)
|
|
6821
6871
|
break;
|
|
6822
6872
|
dependencies.push(...expanded.dependencies);
|
|
@@ -6833,7 +6883,7 @@ async function expandIterativeAsyncWithTrace(source, fetcher, maxIterations) {
|
|
|
6833
6883
|
source: current2,
|
|
6834
6884
|
dependencies,
|
|
6835
6885
|
iterations,
|
|
6836
|
-
reachedMaxIterations: iterations.length === maxIterations &&
|
|
6886
|
+
reachedMaxIterations: iterations.length === maxIterations && hasResolvableDirectives(current2, shouldDefer)
|
|
6837
6887
|
};
|
|
6838
6888
|
}
|
|
6839
6889
|
function expandOneIteration(source, fetcher, replacementCache) {
|
|
@@ -6872,14 +6922,15 @@ function expandOneIterationWithTrace(source, fetcher, replacementCache, iteratio
|
|
|
6872
6922
|
dependencies
|
|
6873
6923
|
};
|
|
6874
6924
|
}
|
|
6875
|
-
async function expandOneIterationAsyncWithTrace(source, fetcher, replacementCache, iteration) {
|
|
6925
|
+
async function expandOneIterationAsyncWithTrace(source, fetcher, replacementCache, iteration, shouldDefer) {
|
|
6876
6926
|
if (!MAYBE_INCLUDE_PATTERN.test(source))
|
|
6877
6927
|
return null;
|
|
6878
6928
|
const directives = scanIncludeDirectives(source);
|
|
6879
6929
|
if (directives.length === 0)
|
|
6880
6930
|
return null;
|
|
6881
6931
|
const references = directives.map(createIncludeReference);
|
|
6882
|
-
const
|
|
6932
|
+
const deferred = references.map((reference) => shouldDefer?.(reference) ?? false);
|
|
6933
|
+
const replacements = await Promise.all(directives.map((directive, index) => deferred[index] ? Promise.resolve(source.slice(directive.start, directive.end)) : replaceCachedAsync(directive.inner, fetcher, replacementCache)));
|
|
6883
6934
|
const parts = [];
|
|
6884
6935
|
let lastPos = 0;
|
|
6885
6936
|
for (let i = 0;i < directives.length; i++) {
|
|
@@ -6891,9 +6942,15 @@ async function expandOneIterationAsyncWithTrace(source, fetcher, replacementCach
|
|
|
6891
6942
|
return {
|
|
6892
6943
|
source: parts.join(""),
|
|
6893
6944
|
references,
|
|
6894
|
-
dependencies: references.map((reference) => ({ ...reference, iteration }))
|
|
6945
|
+
dependencies: references.filter((_, index) => !deferred[index]).map((reference) => ({ ...reference, iteration }))
|
|
6895
6946
|
};
|
|
6896
6947
|
}
|
|
6948
|
+
function hasResolvableDirectives(source, shouldDefer) {
|
|
6949
|
+
const directives = scanIncludeDirectives(source);
|
|
6950
|
+
if (!shouldDefer)
|
|
6951
|
+
return directives.length > 0;
|
|
6952
|
+
return directives.some((directive) => !shouldDefer(createIncludeReference(directive)));
|
|
6953
|
+
}
|
|
6897
6954
|
function replaceCached(inner, fetcher, cache) {
|
|
6898
6955
|
const cached = cache.get(inner);
|
|
6899
6956
|
if (cached !== undefined)
|
|
@@ -6949,6 +7006,19 @@ async function resolveIncludesAsyncWithTrace(source, fetcher, options) {
|
|
|
6949
7006
|
const cachedFetcher = createCachedAsyncIncludeFetcher(fetcher, normalizePageKey);
|
|
6950
7007
|
return expandIterativeAsyncWithTrace(source, cachedFetcher, maxIterations);
|
|
6951
7008
|
}
|
|
7009
|
+
async function resolveIncludesAsyncWithTraceSelective(source, fetcher, shouldDefer, options) {
|
|
7010
|
+
if (options?.settings && !options.settings.enablePageSyntax) {
|
|
7011
|
+
return {
|
|
7012
|
+
source,
|
|
7013
|
+
dependencies: [],
|
|
7014
|
+
iterations: [],
|
|
7015
|
+
reachedMaxIterations: false
|
|
7016
|
+
};
|
|
7017
|
+
}
|
|
7018
|
+
const maxIterations = options?.maxIterations ?? 10;
|
|
7019
|
+
const cachedFetcher = createCachedAsyncIncludeFetcher(fetcher, normalizePageKey);
|
|
7020
|
+
return expandIterativeAsyncWithTrace(source, cachedFetcher, maxIterations, shouldDefer);
|
|
7021
|
+
}
|
|
6952
7022
|
function normalizePageKey(location) {
|
|
6953
7023
|
const site = location.site ?? "";
|
|
6954
7024
|
const page = location.page.toLowerCase();
|
|
@@ -8419,6 +8489,9 @@ var tabviewRule = {
|
|
|
8419
8489
|
}
|
|
8420
8490
|
};
|
|
8421
8491
|
|
|
8492
|
+
// packages/parser/src/parser/rules/block/include/index.ts
|
|
8493
|
+
import { lineBreak, paragraph, text } from "@wdprlib/ast";
|
|
8494
|
+
|
|
8422
8495
|
// packages/parser/src/parser/rules/block/include/arguments.ts
|
|
8423
8496
|
function collectIncludeArguments(ctx, startPos) {
|
|
8424
8497
|
const argumentTokens = [];
|
|
@@ -8534,6 +8607,7 @@ var includeRule = {
|
|
|
8534
8607
|
}
|
|
8535
8608
|
pos++;
|
|
8536
8609
|
consumed++;
|
|
8610
|
+
const directiveEnd = pos;
|
|
8537
8611
|
if (ctx.tokens[pos]?.type === "NEWLINE") {
|
|
8538
8612
|
pos++;
|
|
8539
8613
|
consumed++;
|
|
@@ -8541,6 +8615,20 @@ var includeRule = {
|
|
|
8541
8615
|
if (!args.target) {
|
|
8542
8616
|
return { success: false };
|
|
8543
8617
|
}
|
|
8618
|
+
const location = parsePageRef(args.target);
|
|
8619
|
+
if (ctx.deferInclude?.(location)) {
|
|
8620
|
+
const source = ctx.tokens.slice(ctx.pos, directiveEnd).map((token5) => token5.value).join("");
|
|
8621
|
+
const elements = [];
|
|
8622
|
+
const lines = source.split(`
|
|
8623
|
+
`);
|
|
8624
|
+
for (let index = 0;index < lines.length; index++) {
|
|
8625
|
+
if (index > 0)
|
|
8626
|
+
elements.push(lineBreak());
|
|
8627
|
+
if (lines[index] !== "")
|
|
8628
|
+
elements.push(text(lines[index]));
|
|
8629
|
+
}
|
|
8630
|
+
return { success: true, elements: [paragraph(elements)], consumed };
|
|
8631
|
+
}
|
|
8544
8632
|
return {
|
|
8545
8633
|
success: true,
|
|
8546
8634
|
elements: [
|
|
@@ -8549,7 +8637,7 @@ var includeRule = {
|
|
|
8549
8637
|
data: {
|
|
8550
8638
|
"paragraph-safe": false,
|
|
8551
8639
|
variables: parseVariables(args.argumentTokens),
|
|
8552
|
-
location
|
|
8640
|
+
location,
|
|
8553
8641
|
elements: []
|
|
8554
8642
|
}
|
|
8555
8643
|
}
|
|
@@ -9817,9 +9905,9 @@ function parseGalleryItemLine(content) {
|
|
|
9817
9905
|
const alt = attrs.get("alt") ?? null;
|
|
9818
9906
|
return { source, link, alt, newWindow };
|
|
9819
9907
|
}
|
|
9820
|
-
function parseItemAttrs(
|
|
9908
|
+
function parseItemAttrs(text2) {
|
|
9821
9909
|
const attrs = new Map;
|
|
9822
|
-
const parts =
|
|
9910
|
+
const parts = text2.trim().split('="');
|
|
9823
9911
|
let key = parts[0]?.trim() ?? "";
|
|
9824
9912
|
for (let i = 1;i < parts.length; i++) {
|
|
9825
9913
|
const val = parts[i] ?? "";
|
|
@@ -11881,11 +11969,11 @@ var guillemetRule = {
|
|
|
11881
11969
|
startTokens: ["LEFT_DOUBLE_ANGLE", "RIGHT_DOUBLE_ANGLE"],
|
|
11882
11970
|
parse(ctx) {
|
|
11883
11971
|
const token5 = ctx.tokens[ctx.pos];
|
|
11884
|
-
const
|
|
11885
|
-
if (
|
|
11972
|
+
const text2 = guillemetText(token5?.type);
|
|
11973
|
+
if (text2) {
|
|
11886
11974
|
return {
|
|
11887
11975
|
success: true,
|
|
11888
|
-
elements: [{ element: "text", data:
|
|
11976
|
+
elements: [{ element: "text", data: text2 }],
|
|
11889
11977
|
consumed: 1
|
|
11890
11978
|
};
|
|
11891
11979
|
}
|
|
@@ -12843,7 +12931,7 @@ var inlineRules = [
|
|
|
12843
12931
|
textRule
|
|
12844
12932
|
];
|
|
12845
12933
|
// packages/parser/src/parser/parse/context.ts
|
|
12846
|
-
function createParseContext(tokens, options = {}) {
|
|
12934
|
+
function createParseContext(tokens, options = {}, deferInclude) {
|
|
12847
12935
|
return {
|
|
12848
12936
|
tokens,
|
|
12849
12937
|
pos: 0,
|
|
@@ -12851,6 +12939,7 @@ function createParseContext(tokens, options = {}) {
|
|
|
12851
12939
|
trackPositions: options.trackPositions ?? true,
|
|
12852
12940
|
settings: options.settings ?? DEFAULT_SETTINGS,
|
|
12853
12941
|
appendImplicitFootnoteBlock: options.appendImplicitFootnoteBlock ?? true,
|
|
12942
|
+
deferInclude,
|
|
12854
12943
|
footnotes: [],
|
|
12855
12944
|
tocEntries: [],
|
|
12856
12945
|
codeBlocks: [],
|
|
@@ -13427,39 +13516,41 @@ class Parser {
|
|
|
13427
13516
|
this.ctx = createParseContext(tokens, options);
|
|
13428
13517
|
}
|
|
13429
13518
|
parse() {
|
|
13430
|
-
|
|
13431
|
-
while (!this.isAtEnd()) {
|
|
13432
|
-
const blocks = this.parseBlock();
|
|
13433
|
-
children.push(...blocks);
|
|
13434
|
-
}
|
|
13435
|
-
return finalizeParseResult(this.ctx, children);
|
|
13519
|
+
return parseContext(this.ctx);
|
|
13436
13520
|
}
|
|
13437
|
-
|
|
13438
|
-
|
|
13439
|
-
|
|
13440
|
-
|
|
13441
|
-
|
|
13442
|
-
|
|
13443
|
-
|
|
13444
|
-
|
|
13445
|
-
type: "EOF",
|
|
13446
|
-
value: "",
|
|
13447
|
-
position: {
|
|
13448
|
-
start: { line: 0, column: 0, offset: 0 },
|
|
13449
|
-
end: { line: 0, column: 0, offset: 0 }
|
|
13450
|
-
},
|
|
13451
|
-
lineStart: false
|
|
13452
|
-
};
|
|
13453
|
-
}
|
|
13454
|
-
skipWhitespace() {
|
|
13455
|
-
while (this.currentToken().type === "WHITESPACE") {
|
|
13456
|
-
this.ctx.pos++;
|
|
13457
|
-
}
|
|
13521
|
+
}
|
|
13522
|
+
function parseTokensWithIncludeDeferral(tokens, options, deferInclude) {
|
|
13523
|
+
return parseContext(createParseContext(tokens, options, deferInclude));
|
|
13524
|
+
}
|
|
13525
|
+
function parseContext(ctx) {
|
|
13526
|
+
const children = [];
|
|
13527
|
+
while (!isAtEnd3(ctx)) {
|
|
13528
|
+
children.push(...parseBlock(ctx));
|
|
13458
13529
|
}
|
|
13459
|
-
|
|
13460
|
-
|
|
13530
|
+
return finalizeParseResult(ctx, children);
|
|
13531
|
+
}
|
|
13532
|
+
function isAtEnd3(ctx) {
|
|
13533
|
+
return ctx.pos >= ctx.tokens.length || currentToken2(ctx).type === "EOF";
|
|
13534
|
+
}
|
|
13535
|
+
function currentToken2(ctx) {
|
|
13536
|
+
return ctx.tokens[ctx.pos] ?? {
|
|
13537
|
+
type: "EOF",
|
|
13538
|
+
value: "",
|
|
13539
|
+
position: {
|
|
13540
|
+
start: { line: 0, column: 0, offset: 0 },
|
|
13541
|
+
end: { line: 0, column: 0, offset: 0 }
|
|
13542
|
+
},
|
|
13543
|
+
lineStart: false
|
|
13544
|
+
};
|
|
13545
|
+
}
|
|
13546
|
+
function skipWhitespace4(ctx) {
|
|
13547
|
+
while (currentToken2(ctx).type === "WHITESPACE") {
|
|
13548
|
+
ctx.pos++;
|
|
13461
13549
|
}
|
|
13462
13550
|
}
|
|
13551
|
+
function parseBlock(ctx) {
|
|
13552
|
+
return parseNextBlock(ctx, () => skipWhitespace4(ctx), () => isAtEnd3(ctx));
|
|
13553
|
+
}
|
|
13463
13554
|
|
|
13464
13555
|
// packages/parser/src/parser/parse/plain-non-ascii.ts
|
|
13465
13556
|
var LARGE_PLAIN_TEXT_MIN_LENGTH = 1e5;
|
|
@@ -13467,8 +13558,8 @@ function parsePlainNonAsciiDocument(source) {
|
|
|
13467
13558
|
if (!isPlainNonAsciiDocument(source)) {
|
|
13468
13559
|
return null;
|
|
13469
13560
|
}
|
|
13470
|
-
const
|
|
13471
|
-
const elements =
|
|
13561
|
+
const text2 = trimNewlineEdges(source);
|
|
13562
|
+
const elements = text2 === "" ? [] : buildPlainParagraphs(text2);
|
|
13472
13563
|
elements.push({
|
|
13473
13564
|
element: "footnote-block",
|
|
13474
13565
|
data: { title: null, hide: false }
|
|
@@ -13488,8 +13579,8 @@ function parseLargePlainTextDocument(source) {
|
|
|
13488
13579
|
if (source.length < LARGE_PLAIN_TEXT_MIN_LENGTH || hasWikitextSyntaxCandidate(source)) {
|
|
13489
13580
|
return null;
|
|
13490
13581
|
}
|
|
13491
|
-
const
|
|
13492
|
-
const elements =
|
|
13582
|
+
const text2 = trimNewlineEdges(source);
|
|
13583
|
+
const elements = text2 === "" ? [] : buildPlainParagraphs(text2);
|
|
13493
13584
|
elements.push({
|
|
13494
13585
|
element: "footnote-block",
|
|
13495
13586
|
data: { title: null, hide: false }
|
|
@@ -13554,10 +13645,10 @@ function trimNewlineEdges(source) {
|
|
|
13554
13645
|
end--;
|
|
13555
13646
|
return start === 0 && end === source.length ? source : source.slice(start, end);
|
|
13556
13647
|
}
|
|
13557
|
-
function buildPlainParagraphs(
|
|
13558
|
-
const paragraphs =
|
|
13648
|
+
function buildPlainParagraphs(text2) {
|
|
13649
|
+
const paragraphs = text2.indexOf(`
|
|
13559
13650
|
|
|
13560
|
-
`) === -1 ? [
|
|
13651
|
+
`) === -1 ? [text2] : text2.split(/\n{2,}/).filter((part) => part !== "");
|
|
13561
13652
|
return paragraphs.map((paragraphText) => ({
|
|
13562
13653
|
element: "container",
|
|
13563
13654
|
data: {
|
|
@@ -13567,8 +13658,8 @@ function buildPlainParagraphs(text) {
|
|
|
13567
13658
|
}
|
|
13568
13659
|
}));
|
|
13569
13660
|
}
|
|
13570
|
-
function buildPlainParagraphElements(
|
|
13571
|
-
const lines =
|
|
13661
|
+
function buildPlainParagraphElements(text2) {
|
|
13662
|
+
const lines = text2.split(`
|
|
13572
13663
|
`);
|
|
13573
13664
|
const elements = Array.from({ length: lines.length * 2 - 1 });
|
|
13574
13665
|
let out = 0;
|
|
@@ -13582,32 +13673,32 @@ function buildPlainParagraphElements(text) {
|
|
|
13582
13673
|
}
|
|
13583
13674
|
|
|
13584
13675
|
// packages/parser/src/parser/preprocess/whitespace/detection.ts
|
|
13585
|
-
function needsWhitespaceSubstitution(
|
|
13586
|
-
if (
|
|
13676
|
+
function needsWhitespaceSubstitution(text2) {
|
|
13677
|
+
if (text2.length === 0)
|
|
13587
13678
|
return false;
|
|
13588
|
-
if (
|
|
13589
|
-
` ||
|
|
13679
|
+
if (text2[0] === `
|
|
13680
|
+
` || text2[0] === " " || text2[text2.length - 1] === `
|
|
13590
13681
|
`) {
|
|
13591
13682
|
return true;
|
|
13592
13683
|
}
|
|
13593
|
-
return
|
|
13684
|
+
return text2.indexOf("\r") !== -1 || text2.indexOf("\t") !== -1 || text2.indexOf("\x00") !== -1 || text2.indexOf(" ") !== -1 || text2.indexOf(" ") !== -1 || text2.indexOf("\\\n") !== -1 || text2.indexOf(`
|
|
13594
13685
|
|
|
13595
13686
|
|
|
13596
|
-
`) !== -1 ||
|
|
13687
|
+
`) !== -1 || text2.indexOf(`
|
|
13597
13688
|
`) !== -1;
|
|
13598
13689
|
}
|
|
13599
|
-
function mayContainWhitespaceOnlyLine(
|
|
13600
|
-
const first =
|
|
13690
|
+
function mayContainWhitespaceOnlyLine(text2) {
|
|
13691
|
+
const first = text2[0];
|
|
13601
13692
|
if (first === " " || first === "\t" || first === `
|
|
13602
13693
|
` || first === " " || first === " ") {
|
|
13603
13694
|
return true;
|
|
13604
13695
|
}
|
|
13605
|
-
return
|
|
13606
|
-
`) !== -1 ||
|
|
13696
|
+
return text2.indexOf(`
|
|
13697
|
+
`) !== -1 || text2.indexOf(`
|
|
13607
13698
|
|
|
13608
|
-
`) !== -1 ||
|
|
13609
|
-
`) !== -1 ||
|
|
13610
|
-
`) !== -1 ||
|
|
13699
|
+
`) !== -1 || text2.indexOf(`
|
|
13700
|
+
`) !== -1 || text2.indexOf(`
|
|
13701
|
+
`) !== -1 || text2.indexOf(`
|
|
13611
13702
|
`) !== -1;
|
|
13612
13703
|
}
|
|
13613
13704
|
|
|
@@ -13620,18 +13711,18 @@ var TABS = /\t/g;
|
|
|
13620
13711
|
var NULL_CHARS = new RegExp(String.fromCharCode(0), "g");
|
|
13621
13712
|
|
|
13622
13713
|
// packages/parser/src/parser/preprocess/whitespace/leading-spaces.ts
|
|
13623
|
-
function replaceLeadingSpaces(
|
|
13624
|
-
return
|
|
13714
|
+
function replaceLeadingSpaces(text2) {
|
|
13715
|
+
return text2.replace(LEADING_NONSTANDARD_WHITESPACE, (match) => {
|
|
13625
13716
|
return " ".repeat(match.length);
|
|
13626
13717
|
});
|
|
13627
13718
|
}
|
|
13628
13719
|
|
|
13629
13720
|
// packages/parser/src/parser/preprocess/whitespace/index.ts
|
|
13630
|
-
function substitute(
|
|
13631
|
-
if (!needsWhitespaceSubstitution(
|
|
13632
|
-
return
|
|
13721
|
+
function substitute(text2) {
|
|
13722
|
+
if (!needsWhitespaceSubstitution(text2)) {
|
|
13723
|
+
return text2;
|
|
13633
13724
|
}
|
|
13634
|
-
let result =
|
|
13725
|
+
let result = text2;
|
|
13635
13726
|
if (result.indexOf("\r") !== -1) {
|
|
13636
13727
|
result = result.replace(DOS_MAC_NEWLINES, `
|
|
13637
13728
|
`);
|
|
@@ -13661,21 +13752,21 @@ function substitute(text) {
|
|
|
13661
13752
|
}
|
|
13662
13753
|
return result;
|
|
13663
13754
|
}
|
|
13664
|
-
function trimLeadingNewlines(
|
|
13755
|
+
function trimLeadingNewlines(text2) {
|
|
13665
13756
|
let index = 0;
|
|
13666
|
-
while (
|
|
13757
|
+
while (text2[index] === `
|
|
13667
13758
|
`) {
|
|
13668
13759
|
index++;
|
|
13669
13760
|
}
|
|
13670
|
-
return index === 0 ?
|
|
13761
|
+
return index === 0 ? text2 : text2.slice(index);
|
|
13671
13762
|
}
|
|
13672
|
-
function trimTrailingNewlines(
|
|
13673
|
-
let end =
|
|
13674
|
-
while (end > 0 &&
|
|
13763
|
+
function trimTrailingNewlines(text2) {
|
|
13764
|
+
let end = text2.length;
|
|
13765
|
+
while (end > 0 && text2[end - 1] === `
|
|
13675
13766
|
`) {
|
|
13676
13767
|
end--;
|
|
13677
13768
|
}
|
|
13678
|
-
return end ===
|
|
13769
|
+
return end === text2.length ? text2 : text2.slice(0, end);
|
|
13679
13770
|
}
|
|
13680
13771
|
|
|
13681
13772
|
// packages/parser/src/parser/preprocess/typography.ts
|
|
@@ -13685,50 +13776,50 @@ var LEFT_DOUBLE_QUOTE = "“";
|
|
|
13685
13776
|
var RIGHT_DOUBLE_QUOTE = "”";
|
|
13686
13777
|
var LOW_DOUBLE_QUOTE = "„";
|
|
13687
13778
|
var ELLIPSIS = "…";
|
|
13688
|
-
function replaceExactEllipsisPattern(
|
|
13779
|
+
function replaceExactEllipsisPattern(text2, pattern) {
|
|
13689
13780
|
let searchFrom = 0;
|
|
13690
13781
|
let result = "";
|
|
13691
13782
|
let lastCopied = 0;
|
|
13692
13783
|
const patternLength = pattern.length;
|
|
13693
|
-
while (searchFrom <
|
|
13694
|
-
const index =
|
|
13784
|
+
while (searchFrom < text2.length) {
|
|
13785
|
+
const index = text2.indexOf(pattern, searchFrom);
|
|
13695
13786
|
if (index === -1)
|
|
13696
13787
|
break;
|
|
13697
|
-
const prev = index > 0 ?
|
|
13698
|
-
const next = index + patternLength <
|
|
13788
|
+
const prev = index > 0 ? text2[index - 1] : "";
|
|
13789
|
+
const next = index + patternLength < text2.length ? text2[index + patternLength] : "";
|
|
13699
13790
|
if (prev !== "." && next !== ".") {
|
|
13700
|
-
result +=
|
|
13791
|
+
result += text2.slice(lastCopied, index) + ELLIPSIS;
|
|
13701
13792
|
lastCopied = index + patternLength;
|
|
13702
13793
|
searchFrom = lastCopied;
|
|
13703
13794
|
} else {
|
|
13704
13795
|
searchFrom = index + 1;
|
|
13705
13796
|
}
|
|
13706
13797
|
}
|
|
13707
|
-
return lastCopied === 0 ?
|
|
13798
|
+
return lastCopied === 0 ? text2 : result + text2.slice(lastCopied);
|
|
13708
13799
|
}
|
|
13709
|
-
function replaceDelimitedTypography(
|
|
13800
|
+
function replaceDelimitedTypography(text2, opener, closer, leftQuote, rightQuote) {
|
|
13710
13801
|
let searchFrom = 0;
|
|
13711
13802
|
let result = "";
|
|
13712
13803
|
let lastCopied = 0;
|
|
13713
|
-
while (searchFrom <
|
|
13714
|
-
const openIndex =
|
|
13804
|
+
while (searchFrom < text2.length) {
|
|
13805
|
+
const openIndex = text2.indexOf(opener, searchFrom);
|
|
13715
13806
|
if (openIndex === -1)
|
|
13716
13807
|
break;
|
|
13717
13808
|
const contentStart = openIndex + opener.length;
|
|
13718
|
-
const closeIndex =
|
|
13809
|
+
const closeIndex = text2.indexOf(closer, contentStart);
|
|
13719
13810
|
if (closeIndex === -1)
|
|
13720
13811
|
break;
|
|
13721
|
-
result +=
|
|
13812
|
+
result += text2.slice(lastCopied, openIndex);
|
|
13722
13813
|
result += leftQuote;
|
|
13723
|
-
result +=
|
|
13814
|
+
result += text2.slice(contentStart, closeIndex);
|
|
13724
13815
|
result += rightQuote;
|
|
13725
13816
|
lastCopied = closeIndex + closer.length;
|
|
13726
13817
|
searchFrom = lastCopied;
|
|
13727
13818
|
}
|
|
13728
|
-
return lastCopied === 0 ?
|
|
13819
|
+
return lastCopied === 0 ? text2 : result + text2.slice(lastCopied);
|
|
13729
13820
|
}
|
|
13730
|
-
function substitute2(
|
|
13731
|
-
let result =
|
|
13821
|
+
function substitute2(text2) {
|
|
13822
|
+
let result = text2;
|
|
13732
13823
|
if (result.includes("``") && result.includes("''")) {
|
|
13733
13824
|
result = replaceDelimitedTypography(result, "``", "''", LEFT_DOUBLE_QUOTE, RIGHT_DOUBLE_QUOTE);
|
|
13734
13825
|
}
|
|
@@ -13748,8 +13839,8 @@ function substitute2(text) {
|
|
|
13748
13839
|
}
|
|
13749
13840
|
|
|
13750
13841
|
// packages/parser/src/parser/preprocess/index.ts
|
|
13751
|
-
function preprocess(
|
|
13752
|
-
let result =
|
|
13842
|
+
function preprocess(text2) {
|
|
13843
|
+
let result = text2;
|
|
13753
13844
|
result = substitute(result);
|
|
13754
13845
|
result = substitute2(result);
|
|
13755
13846
|
return result;
|
|
@@ -13961,6 +14052,12 @@ function prepareSourceForParse(source, options) {
|
|
|
13961
14052
|
// packages/parser/src/parser/parse/index.ts
|
|
13962
14053
|
var COMPACT_TEXT_RUN_SOURCE_LENGTH = 1e5;
|
|
13963
14054
|
function parse(source, options) {
|
|
14055
|
+
return parseSource(source, options);
|
|
14056
|
+
}
|
|
14057
|
+
function parseWithIncludeDeferral(source, options, deferInclude) {
|
|
14058
|
+
return parseSource(source, options, deferInclude);
|
|
14059
|
+
}
|
|
14060
|
+
function parseSource(source, options, deferInclude) {
|
|
13964
14061
|
const plainResult = parsePlainNonAsciiDocument(source);
|
|
13965
14062
|
if (plainResult) {
|
|
13966
14063
|
return plainResult;
|
|
@@ -13974,7 +14071,7 @@ function parse(source, options) {
|
|
|
13974
14071
|
trackPositions: options?.trackPositions,
|
|
13975
14072
|
compactTextRuns: preprocessed.length >= COMPACT_TEXT_RUN_SOURCE_LENGTH
|
|
13976
14073
|
});
|
|
13977
|
-
return new Parser(tokens, options).parse();
|
|
14074
|
+
return deferInclude ? parseTokensWithIncludeDeferral(tokens, options ?? {}, deferInclude) : new Parser(tokens, options).parse();
|
|
13978
14075
|
}
|
|
13979
14076
|
// packages/parser/src/pipeline/process.ts
|
|
13980
14077
|
import {
|
|
@@ -14170,31 +14267,32 @@ async function processWikitext(source, options) {
|
|
|
14170
14267
|
const diagnostics = [];
|
|
14171
14268
|
const fetchInclude = createRequestIncludeFetcher(options.dataProvider?.fetchInclude ? (pageRef) => options.dataProvider.fetchInclude(pageRef, callbackContext) : undefined);
|
|
14172
14269
|
const resolveSource = async (input) => {
|
|
14173
|
-
|
|
14174
|
-
|
|
14175
|
-
|
|
14176
|
-
|
|
14177
|
-
|
|
14178
|
-
|
|
14179
|
-
|
|
14180
|
-
|
|
14181
|
-
|
|
14270
|
+
let expanded = input;
|
|
14271
|
+
if (fetchInclude) {
|
|
14272
|
+
const resolution = await resolveIncludesAsyncWithTraceSelective(input, fetchInclude, (reference) => isCurrentPageInclude(reference.location, options.page), {
|
|
14273
|
+
maxIterations: options.includeMaxIterations,
|
|
14274
|
+
settings
|
|
14275
|
+
});
|
|
14276
|
+
dependencies.push(...resolution.dependencies);
|
|
14277
|
+
if (resolution.reachedMaxIterations) {
|
|
14278
|
+
diagnostics.push(createLimitDiagnostic("include-resolution-limit", `Include expansion stopped after ${options.includeMaxIterations ?? 10} iterations.`));
|
|
14279
|
+
}
|
|
14280
|
+
expanded = resolution.source;
|
|
14182
14281
|
}
|
|
14183
|
-
return
|
|
14282
|
+
return expanded;
|
|
14184
14283
|
};
|
|
14185
|
-
const
|
|
14186
|
-
|
|
14187
|
-
|
|
14188
|
-
|
|
14189
|
-
|
|
14190
|
-
|
|
14284
|
+
const parseSource2 = async (input) => {
|
|
14285
|
+
const expanded = await resolveSource(input);
|
|
14286
|
+
return parseWithIncludeDeferral(expanded, {
|
|
14287
|
+
settings,
|
|
14288
|
+
pageTags: options.page.tags,
|
|
14289
|
+
appendImplicitFootnoteBlock: false
|
|
14290
|
+
}, (location) => isCurrentPageInclude(location, options.page));
|
|
14291
|
+
};
|
|
14292
|
+
const initial = await parseSource2(source);
|
|
14191
14293
|
diagnostics.push(...initial.diagnostics);
|
|
14192
14294
|
const dataProvider = createModuleDataProvider(options, callbackContext);
|
|
14193
|
-
const parseFragment =
|
|
14194
|
-
settings,
|
|
14195
|
-
pageTags: options.page.tags,
|
|
14196
|
-
appendImplicitFootnoteBlock: false
|
|
14197
|
-
});
|
|
14295
|
+
const parseFragment = parseSource2;
|
|
14198
14296
|
let ast = initial.ast;
|
|
14199
14297
|
for (let pass = 0;pass < DEFAULT_MODULE_MAX_PASSES; pass++) {
|
|
14200
14298
|
const extraction = extractDataRequirements(ast);
|
|
@@ -14223,6 +14321,14 @@ async function processWikitext(source, options) {
|
|
|
14223
14321
|
dependencies
|
|
14224
14322
|
};
|
|
14225
14323
|
}
|
|
14324
|
+
function isCurrentPageInclude(location, page) {
|
|
14325
|
+
if (location.site !== null) {
|
|
14326
|
+
if (!page.site || location.site.toLowerCase() !== page.site.toLowerCase())
|
|
14327
|
+
return false;
|
|
14328
|
+
}
|
|
14329
|
+
const target = location.page.toLowerCase();
|
|
14330
|
+
return target === page.fullName.toLowerCase() || target === page.unixName?.toLowerCase();
|
|
14331
|
+
}
|
|
14226
14332
|
function hasResolvableRequirements(requirements, provider) {
|
|
14227
14333
|
return Boolean(provider?.fetchListPages && requirements.listPages.length > 0 || provider?.fetchListUsers && requirements.listUsers.length > 0 || provider?.fetchTagCloud && requirements.tagCloud.length > 0);
|
|
14228
14334
|
}
|
|
@@ -14259,7 +14365,7 @@ function createRequestIncludeFetcher(fetcher) {
|
|
|
14259
14365
|
}
|
|
14260
14366
|
export {
|
|
14261
14367
|
tokenize,
|
|
14262
|
-
text,
|
|
14368
|
+
text2 as text,
|
|
14263
14369
|
resolveTagCloud,
|
|
14264
14370
|
resolveModules,
|
|
14265
14371
|
resolveListUsers,
|
|
@@ -14276,14 +14382,14 @@ export {
|
|
|
14276
14382
|
parseDateSelector,
|
|
14277
14383
|
parseCategory,
|
|
14278
14384
|
parse,
|
|
14279
|
-
paragraph,
|
|
14385
|
+
paragraph2 as paragraph,
|
|
14280
14386
|
normalizeQuery,
|
|
14281
14387
|
matchesListPagesSelectors,
|
|
14282
14388
|
listItemSubList,
|
|
14283
14389
|
listItemElements,
|
|
14284
14390
|
list,
|
|
14285
14391
|
link,
|
|
14286
|
-
lineBreak,
|
|
14392
|
+
lineBreak2 as lineBreak,
|
|
14287
14393
|
italics,
|
|
14288
14394
|
isTagCloudModule,
|
|
14289
14395
|
isListUsersModule2 as isListUsersModule,
|