markdown-magic 2.6.0 → 3.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -10
- package/cli.js +5 -82
- package/lib/block-parser-js.test.js +179 -0
- package/lib/block-parser.js +389 -0
- package/lib/{utils/new-parser.test.js → block-parser.test.js} +168 -50
- package/lib/cli.js +234 -0
- package/lib/cli.test.js +409 -0
- package/lib/defaults.js +12 -0
- package/lib/index.js +319 -184
- package/lib/index.test.js +11 -0
- package/lib/options-parser.js +498 -0
- package/lib/options-parser.test.js +1237 -0
- package/lib/process-contents.js +330 -0
- package/lib/process-file.js +34 -0
- package/lib/transforms/code.js +67 -22
- package/lib/transforms/file.js +13 -10
- package/lib/transforms/remote.js +9 -6
- package/lib/transforms/toc.js +136 -64
- package/lib/transforms/wordCount.js +5 -0
- package/lib/utils/fs.js +340 -0
- package/lib/utils/fs.test.js +268 -0
- package/lib/utils/html-to-json/compat.js +42 -0
- package/lib/utils/html-to-json/format.js +64 -0
- package/lib/utils/html-to-json/index.js +37 -0
- package/lib/utils/html-to-json/lexer.js +345 -0
- package/lib/utils/html-to-json/parser.js +146 -0
- package/lib/utils/html-to-json/stringify.js +37 -0
- package/lib/utils/html-to-json/tags.js +171 -0
- package/lib/utils/index.js +19 -0
- package/{cli-utils.js → lib/utils/load-config.js} +2 -6
- package/lib/utils/logs.js +89 -0
- package/lib/utils/md/filters.js +20 -0
- package/lib/utils/md/find-code-blocks.js +80 -0
- package/lib/utils/md/find-date.js +32 -0
- package/lib/utils/md/find-frontmatter.js +94 -0
- package/lib/utils/md/find-frontmatter.test.js +17 -0
- package/lib/utils/md/find-html-tags.js +105 -0
- package/lib/utils/md/find-images.js +102 -0
- package/lib/utils/md/find-links.js +202 -0
- package/lib/utils/md/find-unmatched-html-tags.js +33 -0
- package/lib/utils/md/fixtures/2022-01-22-date-in-filename.md +14 -0
- package/lib/utils/md/fixtures/file-with-frontmatter.md +32 -0
- package/lib/utils/md/fixtures/file-with-links.md +143 -0
- package/lib/utils/md/md.test.js +37 -0
- package/lib/utils/md/parse.js +122 -0
- package/lib/utils/md/utils.js +19 -0
- package/lib/utils/regex-timeout.js +83 -0
- package/lib/utils/regex.js +38 -5
- package/lib/utils/remoteRequest.js +54 -0
- package/lib/utils/syntax.js +79 -0
- package/lib/utils/text.js +260 -0
- package/lib/utils/text.test.js +311 -0
- package/package.json +26 -26
- package/index.js +0 -46
- package/lib/processFile.js +0 -154
- package/lib/transforms/index.js +0 -114
- package/lib/updateContents.js +0 -125
- package/lib/utils/_md.test.js +0 -63
- package/lib/utils/new-parser.js +0 -412
- package/lib/utils/weird-parse.js +0 -230
- package/lib/utils/weird-parse.test.js +0 -217
package/lib/transforms/toc.js
CHANGED
|
@@ -1,10 +1,19 @@
|
|
|
1
1
|
const { transform } = require('@technote-space/doctoc')
|
|
2
|
-
const { removeLeadingAndTrailingLineBreaks } = require('../utils/regex')
|
|
2
|
+
const { removeLeadingAndTrailingLineBreaks, escapeRegexString } = require('../utils/regex')
|
|
3
|
+
const { findMinIndent } = require('../utils/text')
|
|
3
4
|
|
|
4
|
-
module.exports = async function TOC(
|
|
5
|
+
module.exports = async function TOC(api) {
|
|
6
|
+
// console.log("TOC API", api)
|
|
7
|
+
const { options, fileContent, originalFileContent } = api
|
|
8
|
+
//const currentBlock = api.getCurrentBlock()
|
|
9
|
+
const originalBlock = api.getOriginalBlock()
|
|
10
|
+
const { block } = originalBlock
|
|
11
|
+
// console.log('originalBlock', originalBlock)
|
|
12
|
+
const originalContents = originalFileContent
|
|
5
13
|
const opts = options || {}
|
|
14
|
+
const isSub = opts.sub
|
|
6
15
|
opts.firsth1 = (opts.firsth1) ? true : false
|
|
7
|
-
let contents =
|
|
16
|
+
let contents = fileContent
|
|
8
17
|
// console.log('contents', contents)
|
|
9
18
|
|
|
10
19
|
let debugFileMatch
|
|
@@ -12,13 +21,13 @@ module.exports = async function TOC(content, options, config) {
|
|
|
12
21
|
// debugFileMatch = config.originalPath.match(/packages\/analytics\/README\.md/)
|
|
13
22
|
|
|
14
23
|
// https://www.npmjs.com/package/@technote-space/toc-generator
|
|
15
|
-
const
|
|
24
|
+
const tocOptions = {
|
|
16
25
|
// mode: 'github.com', // github.com | bitbucket.org | gitlab.com | nodejs.org | ghost.org (default: github.com)
|
|
17
26
|
isNotitle: true,
|
|
18
27
|
//isCustomMode: true,
|
|
19
28
|
openingComment: '',
|
|
20
29
|
closingComment: '',
|
|
21
|
-
maxHeaderLevel: (opts.maxDepth) ? Number(opts.maxDepth) : 4
|
|
30
|
+
maxHeaderLevel: (opts.maxDepth) ? Number(opts.maxDepth) : 4,
|
|
22
31
|
// maxHeaderLevel: 2, // default: 4
|
|
23
32
|
// title: '**Table of Contents**',
|
|
24
33
|
// isNotitle: true,
|
|
@@ -34,18 +43,19 @@ module.exports = async function TOC(content, options, config) {
|
|
|
34
43
|
// customTemplate: '<p align="center">${ITEMS}</p>',
|
|
35
44
|
// itemTemplate: '<a href="${LINK}">${TEXT}</a>',
|
|
36
45
|
// separator: '<span>|</span>',
|
|
37
|
-
// footer: '',
|
|
38
|
-
}
|
|
39
|
-
|
|
46
|
+
// footer: 'end',
|
|
47
|
+
}
|
|
48
|
+
const t = await transform(contents, tocOptions)
|
|
49
|
+
let outputText = t.wrappedToc || ''
|
|
40
50
|
|
|
41
51
|
if (debugFileMatch) {
|
|
42
|
-
console.log('before firsth1 removal',
|
|
52
|
+
console.log('before firsth1 removal', outputText)
|
|
43
53
|
}
|
|
44
54
|
|
|
45
55
|
/* remove first h1 */
|
|
46
56
|
if (!opts.firsth1) {
|
|
47
|
-
// console.log('
|
|
48
|
-
const lines =
|
|
57
|
+
// console.log('outputText', outputText)
|
|
58
|
+
const lines = outputText.split('\n')
|
|
49
59
|
let firstH1
|
|
50
60
|
let firstLine
|
|
51
61
|
const countOfH1s = lines.reduce((acc, line, index) => {
|
|
@@ -65,23 +75,25 @@ module.exports = async function TOC(content, options, config) {
|
|
|
65
75
|
const firstHeadingData = (firstH1 && Array.isArray(firstH1)) ? firstH1 : []
|
|
66
76
|
const [ lineText, lineIndex, isFirst, matchText ] = firstHeadingData
|
|
67
77
|
|
|
68
|
-
// verify its h1
|
|
69
|
-
const matchTextEscaped = escapeStringRegexp(matchText)
|
|
70
|
-
// console.log('matchTextEscaped', matchTextEscaped)
|
|
71
|
-
// /^#{1}\s+(.*)/
|
|
72
|
-
const FIRST_H1_REGEX = new RegExp(`^#\\s*\\[?${matchTextEscaped}\\]?(?:.*)?`, 'gim')
|
|
73
|
-
// /^<h1\b[^>]*>([\s\S]*?)<\/h1>/
|
|
74
|
-
const FIRST_HTML_H1_REGEX = new RegExp(`^<h1\\b[^>]*>[\\s]*?(${matchTextEscaped})[\\s]*?<\\/h1>`, 'gim')
|
|
75
|
-
// /^(.*)\n={3,}/
|
|
76
|
-
const FIRST_UNDERSCORE_H1 = new RegExp(`^(${matchTextEscaped})\n={3,}`, 'gim')
|
|
77
|
-
|
|
78
78
|
let docHasHeading = false
|
|
79
|
-
if (
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
79
|
+
if (matchText) {
|
|
80
|
+
// verify its h1
|
|
81
|
+
const matchTextEscaped = escapeRegexString(matchText)
|
|
82
|
+
// console.log('matchTextEscaped', matchTextEscaped)
|
|
83
|
+
// /^#{1}\s+(.*)/
|
|
84
|
+
const FIRST_H1_REGEX = new RegExp(`^#\\s*\\[?${matchTextEscaped}\\]?(?:.*)?`, 'gim')
|
|
85
|
+
// /^<h1\b[^>]*>([\s\S]*?)<\/h1>/
|
|
86
|
+
const FIRST_HTML_H1_REGEX = new RegExp(`^<h1\\b[^>]*>[\\s]*?(${matchTextEscaped})[\\s]*?<\\/h1>`, 'gim')
|
|
87
|
+
// /^(.*)\n={3,}/
|
|
88
|
+
const FIRST_UNDERSCORE_H1 = new RegExp(`^(${matchTextEscaped})\n={3,}`, 'gim')
|
|
89
|
+
|
|
90
|
+
if (contents.match(FIRST_H1_REGEX)) {
|
|
91
|
+
docHasHeading = matchText
|
|
92
|
+
} else if (contents.match(FIRST_HTML_H1_REGEX)) {
|
|
93
|
+
docHasHeading = matchText
|
|
94
|
+
} else if (contents.match(FIRST_UNDERSCORE_H1)) {
|
|
95
|
+
docHasHeading = matchText
|
|
96
|
+
}
|
|
85
97
|
}
|
|
86
98
|
|
|
87
99
|
if (debugFileMatch) {
|
|
@@ -103,7 +115,7 @@ module.exports = async function TOC(content, options, config) {
|
|
|
103
115
|
let foundFirstH1
|
|
104
116
|
|
|
105
117
|
if (docHasHeading) {
|
|
106
|
-
|
|
118
|
+
outputText = lines.reduce((acc, line, i) => {
|
|
107
119
|
const isLineEmpty = line === ''
|
|
108
120
|
if (!isLineEmpty && !firstItemWithContent) {
|
|
109
121
|
firstItemWithContent = i
|
|
@@ -127,14 +139,14 @@ module.exports = async function TOC(content, options, config) {
|
|
|
127
139
|
}, []).join('\n')
|
|
128
140
|
}
|
|
129
141
|
|
|
130
|
-
// console.log('
|
|
142
|
+
// console.log('outputText', outputText)
|
|
131
143
|
if (debugFileMatch) {
|
|
132
|
-
console.log('after firsth1 removal',
|
|
144
|
+
console.log('after firsth1 removal', outputText)
|
|
133
145
|
}
|
|
134
146
|
}
|
|
135
147
|
// console.log(t)
|
|
136
148
|
// Fix <h1> with multiple lines
|
|
137
|
-
const spaces =
|
|
149
|
+
const spaces = outputText.match(/\[\n([\s\S]*?)\]/gm)
|
|
138
150
|
if (spaces) {
|
|
139
151
|
const fixed = spaces[0]
|
|
140
152
|
// all new lines
|
|
@@ -142,33 +154,76 @@ module.exports = async function TOC(content, options, config) {
|
|
|
142
154
|
// leading spaces
|
|
143
155
|
.replace(/\[\s+/, '[')
|
|
144
156
|
.trim()
|
|
145
|
-
|
|
157
|
+
outputText = outputText.replace(spaces[0], fixed)
|
|
146
158
|
}
|
|
147
|
-
// console.log(
|
|
148
|
-
|
|
159
|
+
// console.log(outputText)
|
|
160
|
+
outputText = outputText
|
|
149
161
|
.replace(/(.*)\[Table of Contents\]\(.*\)\n?/i, '')
|
|
150
162
|
.replace(/(.*)\[toc\]\(.*\)\n?/i, '')
|
|
151
163
|
|
|
152
164
|
|
|
153
165
|
if (opts.excludeText) {
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
//
|
|
157
|
-
const
|
|
166
|
+
|
|
167
|
+
outputText = excludeTocItem(outputText, opts.excludeText)
|
|
168
|
+
// // (\s+)?-(.*)\[Usage\]\(.*\)
|
|
169
|
+
// const regex = new RegExp(`\(\\s+\)?-(.*)\\[${opts.excludeText}\\]\\(.*\\)`, 'i')
|
|
170
|
+
// // /(\s+)?-(.*)\[Usage\]\(.*\)(\n\s+(.*))+/im
|
|
171
|
+
// const nestedRegex = new RegExp(`\(\\s+\)?-(.*)\\[${opts.excludeText}\\]\\(.*\\)\(\\n\\s+\(.*\)\)+`, 'i')
|
|
158
172
|
|
|
159
|
-
const hasNested = nestedRegex.exec(
|
|
160
|
-
if (hasNested) {
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
} else {
|
|
171
|
-
|
|
173
|
+
// const hasNested = nestedRegex.exec(outputText)
|
|
174
|
+
// if (hasNested) {
|
|
175
|
+
// // Count indentation of spaces
|
|
176
|
+
// const numberOfSpaces = hasNested[1].replace(/\n/g, '').length
|
|
177
|
+
// const subItems = numberOfSpaces + 1
|
|
178
|
+
// // Update regex to only remove sub items
|
|
179
|
+
// const nestedRegexSpaces = new RegExp(`\(\\s+\)?-(.*)\\[${opts.excludeText}\\]\\(.*\\)\(\\n\\s{${subItems},}\(.*\)\)+`, 'i')
|
|
180
|
+
// // console.log('nestedRegexSpaces', nestedRegexSpaces)
|
|
181
|
+
// // If exclude value has nested sections remove them as well.
|
|
182
|
+
// outputText = outputText.replace(nestedRegexSpaces, '')
|
|
183
|
+
// outputText = outputText.replace(regex, '')
|
|
184
|
+
// } else {
|
|
185
|
+
// outputText = outputText.replace(regex, '')
|
|
186
|
+
// }
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/* Sub table of contents */
|
|
190
|
+
if (isSub) {
|
|
191
|
+
// const start = fileContent.indexOf(open.value)
|
|
192
|
+
// const linesBefore = fileContent.substr(0, start).split('\n')
|
|
193
|
+
// const closestHeading = linesBefore.reverse().find((line) => {
|
|
194
|
+
// return line.match((/^#+/))
|
|
195
|
+
// })
|
|
196
|
+
const linesBefore = originalContents.substr(0, block.start).split('\n')
|
|
197
|
+
const closestHeading = linesBefore.reverse().find((line) => {
|
|
198
|
+
return line.match((/^#+/))
|
|
199
|
+
})
|
|
200
|
+
|
|
201
|
+
if (closestHeading) {
|
|
202
|
+
const headingText = closestHeading.replace(/^#*\s*/, '')
|
|
203
|
+
// console.log('BEFORE', linesBefore)
|
|
204
|
+
// console.log('closest parent heading', closestHeading)
|
|
205
|
+
// console.log('headingText', headingText)
|
|
206
|
+
// https://regex101.com/r/MB85zm/1
|
|
207
|
+
|
|
208
|
+
const findSubToc = new RegExp(`^\(\\s+\)?-(.*)\\[${headingText}\\]\\(.*\\)(\\n\\s.*)+`, 'gm')
|
|
209
|
+
// console.log('findSubToc', findSubToc)
|
|
210
|
+
const single = singleLinePattern(headingText)
|
|
211
|
+
// console.log('single', single)
|
|
212
|
+
const subItems = outputText.match(findSubToc)
|
|
213
|
+
if (subItems) {
|
|
214
|
+
const items = subItems[0].replace(single, '').split('\n')
|
|
215
|
+
.filter(Boolean)
|
|
216
|
+
// console.log('items', items)
|
|
217
|
+
const finalItems = items // .slice(1, items.length)
|
|
218
|
+
if (finalItems) {
|
|
219
|
+
const indent = findMinIndent(finalItems.join('\n'))
|
|
220
|
+
// console.log('min indent', indent)
|
|
221
|
+
// console.log('finalItems', finalItems)
|
|
222
|
+
outputText = finalItems.map((thing) => thing.substring(indent)).join('\n')
|
|
223
|
+
} else {
|
|
224
|
+
// console.log('No sub items')
|
|
225
|
+
}
|
|
226
|
+
}
|
|
172
227
|
}
|
|
173
228
|
}
|
|
174
229
|
|
|
@@ -177,7 +232,7 @@ module.exports = async function TOC(content, options, config) {
|
|
|
177
232
|
const text = (opts.collapseText) ? opts.collapseText : 'Table of Contents'
|
|
178
233
|
return `<details>
|
|
179
234
|
<summary>${text}</summary>
|
|
180
|
-
${
|
|
235
|
+
${outputText
|
|
181
236
|
// Replace leading double spaces
|
|
182
237
|
.replace(/^\n\n/, '\n')
|
|
183
238
|
// Replace trailing double spaces
|
|
@@ -185,7 +240,7 @@ ${output
|
|
|
185
240
|
</details>`
|
|
186
241
|
}
|
|
187
242
|
|
|
188
|
-
return
|
|
243
|
+
return outputText.replace(removeLeadingAndTrailingLineBreaks, '')
|
|
189
244
|
}
|
|
190
245
|
|
|
191
246
|
const HTML_TAG = /<([a-zA-Z1-6]+)([^<]+)*(?:>(.*)<\/\1>|\s+\/>)/gim
|
|
@@ -246,21 +301,38 @@ function parseHtmlProps(mdContents) {
|
|
|
246
301
|
return tags
|
|
247
302
|
}
|
|
248
303
|
|
|
249
|
-
function
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
304
|
+
function singleLinePattern(text) {
|
|
305
|
+
/* (\s+)?-(.*)\[Usage\]\(.*\) */
|
|
306
|
+
return new RegExp(`\(\\s+\)?-(.*)\\[${text}\\]\\(.*\\)`, 'i')
|
|
307
|
+
}
|
|
253
308
|
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
309
|
+
function excludeTocItem(str, excludeText) {
|
|
310
|
+
const matchTextEscaped = escapeRegexString(excludeText)
|
|
311
|
+
/* (\s+)?-(.*)\[Usage\]\(.*\) */
|
|
312
|
+
const regex = singleLinePattern(matchTextEscaped) // new RegExp(`\(\\s+\)?-(.*)\\[${matchTextEscaped}\\]\\(.*\\)`, 'i')
|
|
313
|
+
/* /(\s+)?-(.*)\[Usage\]\(.*\)(\n\s+(.*))+/im */
|
|
314
|
+
const nestedRegex = new RegExp(`\(\\s+\)?-(.*)\\[${matchTextEscaped}\\]\\(.*\\)\(\\n\\s+\(.*\)\)+`, 'i')
|
|
315
|
+
|
|
316
|
+
const hasNested = nestedRegex.exec(str)
|
|
317
|
+
if (hasNested) {
|
|
318
|
+
// Count indentation of spaces
|
|
319
|
+
const numberOfSpaces = (hasNested[1] || '').replace(/\n/g, '').length
|
|
320
|
+
const subItems = numberOfSpaces + 1
|
|
321
|
+
// Update regex to only remove sub items
|
|
322
|
+
const nestedRegexSpaces = new RegExp(`\(\\s+\)?-(.*)\\[${matchTextEscaped}\\]\\(.*\\)\(\\n\\s{${subItems},}\(.*\)\)+`, 'i')
|
|
323
|
+
// console.log('nestedRegexSpaces', nestedRegexSpaces)
|
|
324
|
+
// If exclude value has nested sections remove them as well.
|
|
325
|
+
str = str.replace(nestedRegexSpaces, '')
|
|
326
|
+
str = str.replace(regex, '')
|
|
327
|
+
} else {
|
|
328
|
+
str = str.replace(regex, '')
|
|
329
|
+
}
|
|
330
|
+
return str
|
|
259
331
|
}
|
|
260
332
|
|
|
261
|
-
function removeUndefined(
|
|
333
|
+
function removeUndefined(outputText) {
|
|
262
334
|
// remove undefined from new line and start of string if first H1 is missing
|
|
263
|
-
return
|
|
335
|
+
return outputText.replace(/\nundefined/g, '\n-').replace(/^undefined/g, '-')
|
|
264
336
|
}
|
|
265
337
|
|
|
266
338
|
// Alt https://github.com/thlorenz/doctoc
|
package/lib/utils/fs.js
ADDED
|
@@ -0,0 +1,340 @@
|
|
|
1
|
+
const fs = require('fs').promises
|
|
2
|
+
const path = require('path')
|
|
3
|
+
const globrex = require('globrex')
|
|
4
|
+
const isGlob = require('is-glob')
|
|
5
|
+
const isLocalPath = require('is-local-path')
|
|
6
|
+
const { REGEX_REGEX, escapeRegexString } = require('./regex')
|
|
7
|
+
const { dirname, resolve, join } = require('path')
|
|
8
|
+
const { readdir, stat, readFile } = fs
|
|
9
|
+
|
|
10
|
+
const IS_HIDDEN_FILE = /(^|[\\\/])\.[^\\\/\.]/g
|
|
11
|
+
|
|
12
|
+
// https://github.com/lukeed/escalade
|
|
13
|
+
async function escalade(start, callback) {
|
|
14
|
+
let dir = resolve('.', start)
|
|
15
|
+
let tmp, stats = await stat(dir)
|
|
16
|
+
|
|
17
|
+
if (!stats.isDirectory()) {
|
|
18
|
+
dir = dirname(dir)
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
while (true) {
|
|
22
|
+
tmp = await callback(dir, await readdir(dir))
|
|
23
|
+
if (tmp) return resolve(dir, tmp)
|
|
24
|
+
dir = dirname(tmp = dir)
|
|
25
|
+
if (tmp === dir) break;
|
|
26
|
+
}
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function isRegex(thing) {
|
|
30
|
+
return (thing instanceof RegExp)
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
async function writeFile(filePath, content) {
|
|
34
|
+
try {
|
|
35
|
+
await fs.writeFile(filePath, content)
|
|
36
|
+
} catch(e) {
|
|
37
|
+
const dirName = path.dirname(filePath)
|
|
38
|
+
await fs.mkdir(dirName, { recursive: true })
|
|
39
|
+
await fs.writeFile(filePath, content)
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
async function findUp(start, fileName) {
|
|
44
|
+
const file = await escalade(start, (dir, relativePaths) => {
|
|
45
|
+
// console.log('~> dir:', dir);
|
|
46
|
+
// console.log('~> relativePaths:', relativePaths);
|
|
47
|
+
// console.log('---')
|
|
48
|
+
if (typeof fileName === 'string' && relativePaths.includes(fileName)) {
|
|
49
|
+
// will be resolved into absolute
|
|
50
|
+
return fileName
|
|
51
|
+
}
|
|
52
|
+
if (fileName instanceof RegExp) {
|
|
53
|
+
const found = relativePaths.find((relativePath) => relativePath.match(fileName))
|
|
54
|
+
if (found) return found
|
|
55
|
+
}
|
|
56
|
+
})
|
|
57
|
+
return file
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
// https://github.com/lukeed/totalist
|
|
61
|
+
async function totalist(dir, callback, pre='') {
|
|
62
|
+
dir = resolve('.', dir)
|
|
63
|
+
await readdir(dir).then(arr => {
|
|
64
|
+
return Promise.all(arr.map((str) => {
|
|
65
|
+
let abs = join(dir, str)
|
|
66
|
+
return stat(abs).then(stats => {
|
|
67
|
+
return stats.isDirectory() ? totalist(abs, callback, join(pre, str)) : callback(join(pre, str), abs, stats)
|
|
68
|
+
})
|
|
69
|
+
}))
|
|
70
|
+
})
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function combineRegexes(patterns = []) {
|
|
74
|
+
const string = patterns.map((pat) => {
|
|
75
|
+
if (isRegex(pat)) {
|
|
76
|
+
return pat.source
|
|
77
|
+
} else if (typeof pat === 'string' && REGEX_REGEX.test(pat)) {
|
|
78
|
+
const regexInfo = pat.match(REGEX_REGEX)
|
|
79
|
+
console.log('regexInfo', regexInfo)
|
|
80
|
+
// escapeRegexString
|
|
81
|
+
return regexInfo[1]
|
|
82
|
+
} else if (isGlob(pat)) {
|
|
83
|
+
console.log('pat', pat)
|
|
84
|
+
const result = globrex(pat, { globstar: true, extended: true })
|
|
85
|
+
console.log('result', result)
|
|
86
|
+
return result.regex.source
|
|
87
|
+
}
|
|
88
|
+
return pat
|
|
89
|
+
}).join('|')
|
|
90
|
+
console.log('xxxstring', string)
|
|
91
|
+
return new RegExp(string)
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
// alt https://github.com/duniul/clean-modules/blob/33b66bcfb7825e1fa1eb1e296e523ddba374f1ae/src/utils/filesystem.ts#L92
|
|
95
|
+
// Alt https://github.com/AvianFlu/ncp
|
|
96
|
+
async function getFilePaths(dirName, {
|
|
97
|
+
patterns = [],
|
|
98
|
+
ignore = [],
|
|
99
|
+
excludeGitIgnore = false,
|
|
100
|
+
excludeHidden = false
|
|
101
|
+
}) {
|
|
102
|
+
let findPattern
|
|
103
|
+
let ignorePattern
|
|
104
|
+
let filePaths = []
|
|
105
|
+
let gitIgnoreFiles = []
|
|
106
|
+
let gitIgnoreGlobs = []
|
|
107
|
+
|
|
108
|
+
if (patterns && patterns.length) {
|
|
109
|
+
findPattern = combineRegexes(patterns)
|
|
110
|
+
console.log('findPatternfindPatternfindPattern', patterns)
|
|
111
|
+
}
|
|
112
|
+
if (ignore && ignore.length) {
|
|
113
|
+
ignorePattern = combineRegexes(ignore)
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
if (excludeGitIgnore) {
|
|
117
|
+
const gitIgnoreContents = await getGitignoreContents()
|
|
118
|
+
for (let index = 0; index < gitIgnoreContents.length; index++) {
|
|
119
|
+
const ignoreItem = gitIgnoreContents[index]
|
|
120
|
+
// console.log('ignoreItem', ignoreItem)
|
|
121
|
+
if (isGlob(ignoreItem)) {
|
|
122
|
+
gitIgnoreGlobs.push(ignoreItem)
|
|
123
|
+
} else {
|
|
124
|
+
gitIgnoreFiles.push(
|
|
125
|
+
ignoreItem.replace(/^\.\//, '') // normalize relative paths
|
|
126
|
+
)
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
//*
|
|
132
|
+
console.log('findPattern', findPattern)
|
|
133
|
+
console.log('ignorePattern', ignorePattern)
|
|
134
|
+
console.log('gitIgnoreFiles', gitIgnoreFiles)
|
|
135
|
+
console.log('gitIgnoreGlobs', gitIgnoreGlobs)
|
|
136
|
+
// process.exit(1)
|
|
137
|
+
/** */
|
|
138
|
+
|
|
139
|
+
await totalist(dirName, (relativePath, abs, stats) => {
|
|
140
|
+
const absolutePath = `${dirName}/${relativePath}`
|
|
141
|
+
const topLevelDir = relativePath.substring(0, relativePath.indexOf('/'))
|
|
142
|
+
//console.log('absolutePath', absolutePath)
|
|
143
|
+
// console.log('relativePath', relativePath)
|
|
144
|
+
// console.log('topLevelDir', topLevelDir)
|
|
145
|
+
|
|
146
|
+
/* Remove hidden files */
|
|
147
|
+
if (excludeHidden && IS_HIDDEN_FILE.test(relativePath)) {
|
|
148
|
+
return
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/* Remove files in git ignore */
|
|
152
|
+
if (excludeGitIgnore && gitIgnoreFiles.length) {
|
|
153
|
+
if (gitIgnoreFiles.includes(relativePath)) return
|
|
154
|
+
if (gitIgnoreFiles.includes(path.basename(relativePath))) return
|
|
155
|
+
//* // slow
|
|
156
|
+
if (gitIgnoreFiles.some((ignore) => {
|
|
157
|
+
// console.log('ignore', ignore)
|
|
158
|
+
// return relativePath.indexOf(ignore) > -1
|
|
159
|
+
// return relativePath.split('/')[0] === ignore
|
|
160
|
+
return topLevelDir === ignore || relativePath === ignore
|
|
161
|
+
})) {
|
|
162
|
+
return
|
|
163
|
+
}
|
|
164
|
+
/** */
|
|
165
|
+
}
|
|
166
|
+
|
|
167
|
+
/* Remove files in ignore array */
|
|
168
|
+
if (ignorePattern && ignorePattern.test(relativePath)) {
|
|
169
|
+
// Alt checker https://github.com/axtgr/wildcard-match
|
|
170
|
+
return
|
|
171
|
+
}
|
|
172
|
+
|
|
173
|
+
/* If no patterns supplied add all files */
|
|
174
|
+
if (!findPattern) {
|
|
175
|
+
filePaths.push(absolutePath)
|
|
176
|
+
return
|
|
177
|
+
}
|
|
178
|
+
|
|
179
|
+
/* If pattern match add file! */
|
|
180
|
+
// Alt match https://github.com/micromatch/picomatch
|
|
181
|
+
if (findPattern.test(absolutePath)) {
|
|
182
|
+
// console.log('Match absolutePath', absolutePath)
|
|
183
|
+
filePaths.push(absolutePath)
|
|
184
|
+
return
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/* If pattern match add file! */
|
|
188
|
+
if (findPattern.test(relativePath)) {
|
|
189
|
+
// console.log('Match relativePath', relativePath)
|
|
190
|
+
filePaths.push(absolutePath)
|
|
191
|
+
return
|
|
192
|
+
}
|
|
193
|
+
|
|
194
|
+
/*
|
|
195
|
+
let ignored = false
|
|
196
|
+
for (let index = 0; index < ignore.length; index++) {
|
|
197
|
+
const pattern = ignore[index]
|
|
198
|
+
if (pattern.test(absolutePath)) {
|
|
199
|
+
ignored = true
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
if (!ignored) {
|
|
203
|
+
filePaths.push(absolutePath)
|
|
204
|
+
}
|
|
205
|
+
/** */
|
|
206
|
+
})
|
|
207
|
+
|
|
208
|
+
if (gitIgnoreGlobs && gitIgnoreGlobs.length) {
|
|
209
|
+
console.log('gitIgnoreGlobs', gitIgnoreGlobs)
|
|
210
|
+
let removeFiles = []
|
|
211
|
+
for (let index = 0; index < gitIgnoreGlobs.length; index++) {
|
|
212
|
+
const glob = gitIgnoreGlobs[index]
|
|
213
|
+
const result = globrex(glob) // alt lib https://github.com/axtgr/wildcard-match
|
|
214
|
+
// console.log('result', result)
|
|
215
|
+
for (let n = 0; n < filePaths.length; n++) {
|
|
216
|
+
const file = filePaths[n]
|
|
217
|
+
if (result.regex.test(file)) {
|
|
218
|
+
removeFiles.push(file)
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
/* Remove files that match glob pattern */
|
|
223
|
+
if (removeFiles.length) {
|
|
224
|
+
filePaths = filePaths.filter(function(el) {
|
|
225
|
+
return removeFiles.indexOf(el) < 0
|
|
226
|
+
})
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
|
|
230
|
+
return filePaths
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function convertToRelativePath(file, cwd) {
|
|
234
|
+
return file.replace(cwd, '').replace(/^\//, '')
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
async function getGitignoreContents(filePath = '.gitignore') {
|
|
238
|
+
try {
|
|
239
|
+
const gitIgnoreContent = await readFile(filePath, { encoding: 'utf8' })
|
|
240
|
+
return gitIgnoreContent
|
|
241
|
+
.split(/\r?\n/)
|
|
242
|
+
.filter((line) => !/^\s*$/.test(line) && !/^\s*#/.test(line))
|
|
243
|
+
.map((line) => line.trim().replace(/^\/+|\/+$/g, ''))
|
|
244
|
+
} catch (_a) {
|
|
245
|
+
return []
|
|
246
|
+
}
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// slash at the beginning of a filename
|
|
250
|
+
const leadingPathSeparator = new RegExp(`^${escapeRegexString(path.sep)}`)
|
|
251
|
+
const windowsLeadingPathSeparator = new RegExp('^/')
|
|
252
|
+
|
|
253
|
+
// all slashes in the filename. path.sep is OS agnostic (windows, mac, etc)
|
|
254
|
+
const pathSeparator = new RegExp(escapeRegexString(path.sep), 'g')
|
|
255
|
+
const windowsPathSeparator = new RegExp('/', 'g')
|
|
256
|
+
|
|
257
|
+
// handle MS Windows style double-backslashed filenames
|
|
258
|
+
const windowsDoubleSlashSeparator = new RegExp('\\\\', 'g')
|
|
259
|
+
|
|
260
|
+
// derive `foo.bar.baz` object key from `foo/bar/baz.yml` filename
|
|
261
|
+
function fileNameToKey(filename) {
|
|
262
|
+
// const extension = new RegExp(`${path.extname(filename)}$`)
|
|
263
|
+
const key = filename
|
|
264
|
+
// .replace(extension, '')
|
|
265
|
+
.replace(leadingPathSeparator, '')
|
|
266
|
+
.replace(windowsLeadingPathSeparator, '')
|
|
267
|
+
.replace(pathSeparator, '.')
|
|
268
|
+
.replace(windowsPathSeparator, '.')
|
|
269
|
+
.replace(windowsDoubleSlashSeparator, '.')
|
|
270
|
+
|
|
271
|
+
return key
|
|
272
|
+
}
|
|
273
|
+
|
|
274
|
+
// https://github.com/regexhq/unc-path-regex/blob/master/index.js
|
|
275
|
+
function isUncPath(filepath) {
|
|
276
|
+
return /^[\\\/]{2,}[^\\\/]+[\\\/]+[^\\\/]+/.test(filepath)
|
|
277
|
+
}
|
|
278
|
+
function isRelative(filepath) {
|
|
279
|
+
const isRel = !isUncPath(filepath) && !/^([a-z]:)?[\\\/]/i.test(filepath)
|
|
280
|
+
// console.log(`isRel ${filepath}`, isRel)
|
|
281
|
+
return isRel
|
|
282
|
+
}
|
|
283
|
+
/* Find common parts of 2 paths */
|
|
284
|
+
function resolveCommonParent(mainDir = '', fileDir = '') {
|
|
285
|
+
const parts = mainDir.split('/')
|
|
286
|
+
let acc = ''
|
|
287
|
+
let value = ''
|
|
288
|
+
for (let i = 0; i < parts.length; i++) {
|
|
289
|
+
const element = parts[i];
|
|
290
|
+
acc+= ((i) ? '/' : '') + element
|
|
291
|
+
if (fileDir.startsWith(acc)) {
|
|
292
|
+
value = acc
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
return value
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
function resolveOutputPath(cwd, outputDir, file) {
|
|
299
|
+
// console.log('file', file)
|
|
300
|
+
const fileCommon = resolveCommonParent(cwd, file)
|
|
301
|
+
// console.log('fileCommon', fileCommon)
|
|
302
|
+
const remove = resolveCommonParent(outputDir, file)
|
|
303
|
+
const fileName = file.replace(remove, '').replace(fileCommon, '')
|
|
304
|
+
let outputFilePath = path.join(outputDir, fileName)
|
|
305
|
+
if (isRelative(outputDir)) {
|
|
306
|
+
outputFilePath = path.join(cwd, outputDir, fileName)
|
|
307
|
+
}
|
|
308
|
+
// console.log('isRelative(outputDir)', isRelative(outputDir))
|
|
309
|
+
// console.log('outputDir', outputDir)
|
|
310
|
+
// console.log('fileName', fileName)
|
|
311
|
+
// console.log('remove', remove)
|
|
312
|
+
return outputFilePath
|
|
313
|
+
}
|
|
314
|
+
|
|
315
|
+
function resolveFlatPath(cwd, outputDir, file) {
|
|
316
|
+
/* old setup */
|
|
317
|
+
const fileName = path.basename(file)
|
|
318
|
+
let outputFilePath = path.join(outputDir, fileName)
|
|
319
|
+
if (isRelative(outputDir)) {
|
|
320
|
+
outputFilePath = path.join(cwd, outputDir, fileName)
|
|
321
|
+
}
|
|
322
|
+
return outputFilePath
|
|
323
|
+
}
|
|
324
|
+
|
|
325
|
+
function depth(string) {
|
|
326
|
+
return path.normalize(string).split(path.sep).length - 1;
|
|
327
|
+
}
|
|
328
|
+
|
|
329
|
+
module.exports = {
|
|
330
|
+
isLocalPath,
|
|
331
|
+
writeFile,
|
|
332
|
+
readFile,
|
|
333
|
+
findUp,
|
|
334
|
+
getFilePaths,
|
|
335
|
+
resolveOutputPath,
|
|
336
|
+
resolveFlatPath,
|
|
337
|
+
resolveCommonParent,
|
|
338
|
+
getGitignoreContents,
|
|
339
|
+
convertToRelativePath
|
|
340
|
+
}
|