hdoc-tools 0.9.7 → 0.9.9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/hdoc-build-db.js +4 -2
- package/hdoc-build.js +20 -8
- package/hdoc-validate.js +43 -29
- package/package.json +1 -1
package/hdoc-build-db.js
CHANGED
@@ -14,7 +14,8 @@
|
|
14
14
|
'doc_title',
|
15
15
|
'doc_content',
|
16
16
|
'doc_preview UNINDEXED',
|
17
|
-
'doc_family_id'
|
17
|
+
'doc_family_id',
|
18
|
+
'doc_md5_hash UNINDEXED'
|
18
19
|
],
|
19
20
|
hdoc_meta: [
|
20
21
|
'resource_url',
|
@@ -89,7 +90,8 @@
|
|
89
90
|
index_records[i].index_html.fm_props.title,
|
90
91
|
index_records[i].index_html.text,
|
91
92
|
index_records[i].index_html.preview,
|
92
|
-
book_config.productFamily
|
93
|
+
book_config.productFamily,
|
94
|
+
index_records[i].md5
|
93
95
|
];
|
94
96
|
const index_response = await hdoc_index.insert_record(db, 'hdoc_index', db_schema.hdoc_index, index_vals);
|
95
97
|
if (!index_response.success) {
|
package/hdoc-build.js
CHANGED
@@ -45,6 +45,7 @@
|
|
45
45
|
includes_failed = 0,
|
46
46
|
index_records = [],
|
47
47
|
md_files = [],
|
48
|
+
md_files_delete = [],
|
48
49
|
static_html_files = [],
|
49
50
|
work_path_content = '',
|
50
51
|
verbose = false;
|
@@ -246,7 +247,8 @@
|
|
246
247
|
index_html: hdoc_index.transform_html_for_index(html_txt),
|
247
248
|
metadata: metadata,
|
248
249
|
contributors: contribs,
|
249
|
-
pdf_size: pdf_size
|
250
|
+
pdf_size: pdf_size,
|
251
|
+
md5: file_path.hash
|
250
252
|
});
|
251
253
|
// Save HTML into HTML file
|
252
254
|
try {
|
@@ -259,6 +261,7 @@
|
|
259
261
|
|
260
262
|
const transform_markdown_and_save_html = async function (file_path) {
|
261
263
|
conversion_attempted++;
|
264
|
+
|
262
265
|
if (fs.existsSync(file_path.path)) {
|
263
266
|
// Load markdown file
|
264
267
|
let md_txt = hdoc.expand_variables(fs.readFileSync(file_path.path, 'utf8'));
|
@@ -465,15 +468,13 @@
|
|
465
468
|
index_html: index_details,
|
466
469
|
metadata: metadata,
|
467
470
|
contributors: contribs,
|
468
|
-
pdf_size: pdf_size
|
471
|
+
pdf_size: pdf_size,
|
472
|
+
md5: file_path.hash
|
469
473
|
});
|
470
474
|
|
471
|
-
//
|
472
|
-
|
473
|
-
|
474
|
-
} catch (e) {
|
475
|
-
console.log(`Error deleting ${file_path.path}: ${e}`);
|
476
|
-
}
|
475
|
+
// Add MD file to delete queue
|
476
|
+
md_files_delete.push(file_path.path)
|
477
|
+
|
477
478
|
conversion_success++;
|
478
479
|
return true;
|
479
480
|
}
|
@@ -553,6 +554,7 @@
|
|
553
554
|
};
|
554
555
|
|
555
556
|
const dreeOptions = {
|
557
|
+
hash: true,
|
556
558
|
extensions: ['md', 'html', 'htm'],
|
557
559
|
normalize: true,
|
558
560
|
};
|
@@ -789,6 +791,16 @@
|
|
789
791
|
process.exit(1);
|
790
792
|
}
|
791
793
|
|
794
|
+
// Delete markdown files
|
795
|
+
for (let i = 0; i < md_files_delete.length; i++) {
|
796
|
+
// Delete MD file from _work path
|
797
|
+
try {
|
798
|
+
fs.unlinkSync(md_files_delete[i]);
|
799
|
+
} catch (e) {
|
800
|
+
console.log(`Error deleting ${md_files_delete[i]}: ${e}`);
|
801
|
+
}
|
802
|
+
}
|
803
|
+
|
792
804
|
// Add book read timing to the hdocbook.json
|
793
805
|
hdocbook_config.readingTime = Math.ceil(book_read_time + ((book_read_time / 100) * 10));
|
794
806
|
try {
|
package/hdoc-validate.js
CHANGED
@@ -5,7 +5,6 @@
|
|
5
5
|
cheerio = require('cheerio'),
|
6
6
|
dree = require('dree'),
|
7
7
|
fs = require('fs'),
|
8
|
-
html2text = require('html-to-text'),
|
9
8
|
https = require('https'),
|
10
9
|
path = require('path'),
|
11
10
|
hdoc = require(path.join(__dirname, 'hdoc-module.js')),
|
@@ -26,7 +25,8 @@
|
|
26
25
|
messages = {},
|
27
26
|
warnings = {},
|
28
27
|
errorcount = 0,
|
29
|
-
|
28
|
+
html_to_validate = [],
|
29
|
+
md_to_validate = [],
|
30
30
|
exclude_links = {},
|
31
31
|
exclude_spellcheck = {};
|
32
32
|
|
@@ -45,14 +45,9 @@
|
|
45
45
|
return true;
|
46
46
|
};
|
47
47
|
|
48
|
-
const
|
49
|
-
const
|
50
|
-
const
|
51
|
-
wordwrap: null,
|
52
|
-
selectors: [
|
53
|
-
{ selector: 'code', format: 'skip'}
|
54
|
-
]
|
55
|
-
});
|
48
|
+
const spellcheckContent = async function (sourceFile, excludes) {
|
49
|
+
const text = fs.readFileSync(sourceFile.path, 'utf8');
|
50
|
+
const source_path = sourceFile.relativePath.replace('.'+sourceFile.extension, '');
|
56
51
|
const translate_output = translator.translate(text, spellcheck_options);
|
57
52
|
if(Object.keys(translate_output).length){
|
58
53
|
for (const key in translate_output) {
|
@@ -61,10 +56,10 @@
|
|
61
56
|
for (let i = 0; i < translate_output[key].length; i++) {
|
62
57
|
for (const spelling in translate_output[key][i]) {
|
63
58
|
if (translate_output[key][i].hasOwnProperty(spelling)) {
|
64
|
-
if (!excludes[
|
65
|
-
errors[
|
66
|
-
} else if (!excludes[
|
67
|
-
errors[
|
59
|
+
if (!excludes[source_path]) {
|
60
|
+
errors[sourceFile.relativePath].push(`${error_message} ${spelling} should be ${translate_output[key][i][spelling].details}`);
|
61
|
+
} else if (!excludes[source_path].includes(spelling.toLowerCase())) {
|
62
|
+
errors[sourceFile.relativePath].push(`${error_message} ${spelling} should be ${translate_output[key][i][spelling].details}`);
|
68
63
|
}
|
69
64
|
}
|
70
65
|
}
|
@@ -197,7 +192,7 @@
|
|
197
192
|
const dreeOptions = {
|
198
193
|
descendants: true,
|
199
194
|
depth: 10,
|
200
|
-
extensions: ['htm', 'html'],
|
195
|
+
extensions: ['htm', 'html','md'],
|
201
196
|
hash: false,
|
202
197
|
normalize: true,
|
203
198
|
size: false,
|
@@ -208,7 +203,11 @@
|
|
208
203
|
|
209
204
|
// File scan callback
|
210
205
|
const fileCallback = function (element) {
|
211
|
-
|
206
|
+
if (element.extension.toLowerCase() === 'md') {
|
207
|
+
md_to_validate.push(element);
|
208
|
+
} else {
|
209
|
+
html_to_validate.push(element);
|
210
|
+
}
|
212
211
|
};
|
213
212
|
|
214
213
|
const isRelativePath = function (source_path, html_path, relative_path) {
|
@@ -338,32 +337,47 @@
|
|
338
337
|
}
|
339
338
|
}
|
340
339
|
|
340
|
+
// Do spellchecking on markdown files
|
341
|
+
let md_files_spellchecked = {};
|
342
|
+
for (let i = 0; i < md_to_validate.length; i++) {
|
343
|
+
// Initiate maps for errors and verbose messages for markdown file
|
344
|
+
errors[md_to_validate[i].relativePath] = [];
|
345
|
+
messages[md_to_validate[i].relativePath] = [];
|
346
|
+
warnings[md_to_validate[i].relativePath] = [];
|
347
|
+
|
348
|
+
await spellcheckContent(md_to_validate[i], exclude_spellcheck);
|
349
|
+
md_files_spellchecked[md_to_validate[i].relativePath.replace('.' + md_to_validate[i].extension, '')] = true;
|
350
|
+
}
|
351
|
+
|
352
|
+
// Perform rest of validation against HTML files
|
341
353
|
let listContent = '';
|
342
|
-
for (let i = 0; i <
|
354
|
+
for (let i = 0; i < html_to_validate.length; i++) {
|
343
355
|
|
344
356
|
// Initiate maps for errors and verbose messages for HTML file
|
345
|
-
errors[
|
346
|
-
messages[
|
347
|
-
warnings[
|
357
|
+
errors[html_to_validate[i].relativePath] = [];
|
358
|
+
messages[html_to_validate[i].relativePath] = [];
|
359
|
+
warnings[html_to_validate[i].relativePath] = [];
|
348
360
|
|
349
|
-
// Check for
|
350
|
-
|
361
|
+
// Check for British spellings in static HTML content
|
362
|
+
if (!md_files_spellchecked[html_to_validate[i].relativePath.replace('.' + html_to_validate[i].extension, '')]) {
|
363
|
+
await spellcheckContent(html_to_validate[i], exclude_spellcheck);
|
364
|
+
}
|
351
365
|
|
352
|
-
const links = getLinks(
|
366
|
+
const links = getLinks(html_to_validate[i]);
|
353
367
|
if (links.href.length === 0) {
|
354
|
-
messages[
|
368
|
+
messages[html_to_validate[i].relativePath].push('No links found in file');
|
355
369
|
} else {
|
356
|
-
await checkLinks(source_path,
|
370
|
+
await checkLinks(source_path, html_to_validate[i], links.href, hdocbook_config);
|
357
371
|
}
|
358
372
|
if (links.img.length === 0) {
|
359
|
-
messages[
|
373
|
+
messages[html_to_validate[i].relativePath].push('No images found in file');
|
360
374
|
} else {
|
361
|
-
await checkImages(source_path,
|
375
|
+
await checkImages(source_path, html_to_validate[i], links.img);
|
362
376
|
}
|
363
377
|
|
364
378
|
// Build list content for Google
|
365
|
-
listContent += `/${
|
366
|
-
if (i <
|
379
|
+
listContent += `/${html_to_validate[i].relativePath.replace(path.extname(html_to_validate[i].relativePath), '')}`;
|
380
|
+
if (i < html_to_validate.length - 1) {
|
367
381
|
listContent += '\r\n';
|
368
382
|
}
|
369
383
|
}
|