extract-pdf 0.1.21 → 0.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +74 -74
- package/package.json +1 -1
- package/src/models/annotation.ts +41 -41
- package/src/models/block-type.ts +203 -203
- package/src/models/line-converter.ts +224 -224
- package/src/models/metadata.ts +29 -29
- package/src/models/page.ts +16 -16
- package/src/models/parse-result.ts +32 -32
- package/src/models/parsed-elements.ts +29 -29
- package/src/models/stashing-stream.ts +86 -86
- package/src/models/text-item-line-grouper.ts +41 -41
- package/src/models/word.ts +31 -31
- package/src/pdf-to-html.ts +225 -225
- package/src/transforms/base/to-line-item-block-transform.ts +29 -29
- package/src/transforms/base/to-line-item-transform.ts +29 -29
- package/src/transforms/base/to-text-item-transform.ts +28 -28
- package/src/transforms/block/detect-code-quote-blocks.ts +57 -57
- package/src/transforms/block/detect-list-levels.ts +64 -64
- package/src/transforms/block/gather-blocks.ts +113 -113
- package/src/transforms/calculate-global-stats.ts +132 -132
- package/src/transforms/line-item/compact-lines.ts +92 -92
- package/src/transforms/line-item/detect-headers.ts +173 -173
- package/src/transforms/line-item/detect-list-items.ts +68 -68
- package/src/transforms/line-item/detect-toc.ts +459 -459
- package/src/transforms/line-item/remove-repetitive-elements.ts +101 -101
- package/src/transforms/line-item/vertical-to-horizontal.ts +90 -90
- package/src/transforms/to-html.ts +46 -46
- package/src/utils/is-url-pdf.ts +33 -33
- package/src/utils/page-item-functions.ts +35 -35
- package/src/utils/string-functions.ts +124 -124
|
@@ -1,86 +1,86 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @description Abstract stream processor that buffers ("stashes") consecutive
|
|
3
|
-
* items matching a predicate before flushing them as a group. Subclasses
|
|
4
|
-
* implement `shouldStash`, `doMatchesStash`, and `doFlushStash` to define what
|
|
5
|
-
* constitutes a run. Used by `WordDetectionStream` in `LineConverter` (grouping
|
|
6
|
-
* same-format text spans) and `VerticalsStream` in `VerticalToHorizontal`
|
|
7
|
-
* (merging single-character vertical lines into one horizontal line).
|
|
8
|
-
*/
|
|
9
|
-
// Abstract stream which allows stash items temporarily
|
|
10
|
-
export default class StashingStream {
|
|
11
|
-
results: any[];
|
|
12
|
-
stash: any[];
|
|
13
|
-
|
|
14
|
-
constructor() {
|
|
15
|
-
if (this.constructor === StashingStream) {
|
|
16
|
-
throw new TypeError("Can not construct abstract class.");
|
|
17
|
-
}
|
|
18
|
-
this.results = [];
|
|
19
|
-
this.stash = [];
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
consumeAll(items: any[]): void {
|
|
23
|
-
items.forEach((item) => this.consume(item));
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
consume(item: any): void {
|
|
27
|
-
if (this.shouldStash(item)) {
|
|
28
|
-
if (!this.matchesStash(item)) {
|
|
29
|
-
this.flushStash();
|
|
30
|
-
}
|
|
31
|
-
this.pushOnStash(item);
|
|
32
|
-
} else {
|
|
33
|
-
if (this.stash.length > 0) {
|
|
34
|
-
this.flushStash();
|
|
35
|
-
}
|
|
36
|
-
this.results.push(item);
|
|
37
|
-
}
|
|
38
|
-
}
|
|
39
|
-
|
|
40
|
-
pushOnStash(item: any): void {
|
|
41
|
-
this.onPushOnStash(item);
|
|
42
|
-
this.stash.push(item);
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
complete(): any[] {
|
|
46
|
-
if (this.stash.length > 0) {
|
|
47
|
-
this.flushStash();
|
|
48
|
-
}
|
|
49
|
-
return this.results;
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
matchesStash(item: any): boolean {
|
|
53
|
-
if (this.stash.length === 0) {
|
|
54
|
-
return true;
|
|
55
|
-
}
|
|
56
|
-
const lastItem = this.stash[this.stash.length - 1];
|
|
57
|
-
return this.doMatchesStash(lastItem, item);
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
flushStash(): void {
|
|
61
|
-
if (this.stash.length > 0) {
|
|
62
|
-
this.doFlushStash(this.stash, this.results);
|
|
63
|
-
this.stash = [];
|
|
64
|
-
}
|
|
65
|
-
}
|
|
66
|
-
|
|
67
|
-
onPushOnStash(item: any): void {
|
|
68
|
-
// sub-classes may override
|
|
69
|
-
}
|
|
70
|
-
|
|
71
|
-
shouldStash(item: any): boolean {
|
|
72
|
-
throw new TypeError(" Do not call abstract method foo from child." + item);
|
|
73
|
-
}
|
|
74
|
-
|
|
75
|
-
doMatchesStash(lastItem: any, item: any): boolean {
|
|
76
|
-
throw new TypeError(
|
|
77
|
-
" Do not call abstract method foo from child." + lastItem + item,
|
|
78
|
-
);
|
|
79
|
-
}
|
|
80
|
-
|
|
81
|
-
doFlushStash(stash: any[], results: any[]): void {
|
|
82
|
-
throw new TypeError(
|
|
83
|
-
" Do not call abstract method foo from child." + stash + results,
|
|
84
|
-
);
|
|
85
|
-
}
|
|
86
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* @description Abstract stream processor that buffers ("stashes") consecutive
|
|
3
|
+
* items matching a predicate before flushing them as a group. Subclasses
|
|
4
|
+
* implement `shouldStash`, `doMatchesStash`, and `doFlushStash` to define what
|
|
5
|
+
* constitutes a run. Used by `WordDetectionStream` in `LineConverter` (grouping
|
|
6
|
+
* same-format text spans) and `VerticalsStream` in `VerticalToHorizontal`
|
|
7
|
+
* (merging single-character vertical lines into one horizontal line).
|
|
8
|
+
*/
|
|
9
|
+
// Abstract stream which allows stash items temporarily
|
|
10
|
+
export default class StashingStream {
|
|
11
|
+
results: any[];
|
|
12
|
+
stash: any[];
|
|
13
|
+
|
|
14
|
+
constructor() {
|
|
15
|
+
if (this.constructor === StashingStream) {
|
|
16
|
+
throw new TypeError("Can not construct abstract class.");
|
|
17
|
+
}
|
|
18
|
+
this.results = [];
|
|
19
|
+
this.stash = [];
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
consumeAll(items: any[]): void {
|
|
23
|
+
items.forEach((item) => this.consume(item));
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
consume(item: any): void {
|
|
27
|
+
if (this.shouldStash(item)) {
|
|
28
|
+
if (!this.matchesStash(item)) {
|
|
29
|
+
this.flushStash();
|
|
30
|
+
}
|
|
31
|
+
this.pushOnStash(item);
|
|
32
|
+
} else {
|
|
33
|
+
if (this.stash.length > 0) {
|
|
34
|
+
this.flushStash();
|
|
35
|
+
}
|
|
36
|
+
this.results.push(item);
|
|
37
|
+
}
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
pushOnStash(item: any): void {
|
|
41
|
+
this.onPushOnStash(item);
|
|
42
|
+
this.stash.push(item);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
complete(): any[] {
|
|
46
|
+
if (this.stash.length > 0) {
|
|
47
|
+
this.flushStash();
|
|
48
|
+
}
|
|
49
|
+
return this.results;
|
|
50
|
+
}
|
|
51
|
+
|
|
52
|
+
matchesStash(item: any): boolean {
|
|
53
|
+
if (this.stash.length === 0) {
|
|
54
|
+
return true;
|
|
55
|
+
}
|
|
56
|
+
const lastItem = this.stash[this.stash.length - 1];
|
|
57
|
+
return this.doMatchesStash(lastItem, item);
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
flushStash(): void {
|
|
61
|
+
if (this.stash.length > 0) {
|
|
62
|
+
this.doFlushStash(this.stash, this.results);
|
|
63
|
+
this.stash = [];
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
onPushOnStash(item: any): void {
|
|
68
|
+
// sub-classes may override
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
shouldStash(item: any): boolean {
|
|
72
|
+
throw new TypeError(" Do not call abstract method foo from child." + item);
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
doMatchesStash(lastItem: any, item: any): boolean {
|
|
76
|
+
throw new TypeError(
|
|
77
|
+
" Do not call abstract method foo from child." + lastItem + item,
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
doFlushStash(stash: any[], results: any[]): void {
|
|
82
|
+
throw new TypeError(
|
|
83
|
+
" Do not call abstract method foo from child." + stash + results,
|
|
84
|
+
);
|
|
85
|
+
}
|
|
86
|
+
}
|
|
@@ -1,41 +1,41 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @description Groups a flat array of `TextItem`s into per-line buckets by
|
|
3
|
-
* clustering items whose y-coordinates differ by less than half of
|
|
4
|
-
* `mostUsedDistance`. Each bucket is then sorted by x so items read left-to-right,
|
|
5
|
-
* which is critical for footnote detection and word ordering in `CompactLines`.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
import TextItem from "./text-item";
|
|
9
|
-
import { sortByX } from "../utils/page-item-functions";
|
|
10
|
-
|
|
11
|
-
// Groups all text items which are on the same y line
|
|
12
|
-
export default class TextItemLineGrouper {
|
|
13
|
-
mostUsedDistance: number;
|
|
14
|
-
|
|
15
|
-
constructor(options: { mostUsedDistance?: number }) {
|
|
16
|
-
this.mostUsedDistance = options.mostUsedDistance || 12;
|
|
17
|
-
}
|
|
18
|
-
|
|
19
|
-
group(textItems: TextItem[]): TextItem[][] {
|
|
20
|
-
const lines: TextItem[][] = [];
|
|
21
|
-
var currentLine: TextItem[] = [];
|
|
22
|
-
textItems.forEach((item) => {
|
|
23
|
-
if (
|
|
24
|
-
currentLine.length > 0 &&
|
|
25
|
-
Math.abs(currentLine[0].y - item.y) >= this.mostUsedDistance / 2
|
|
26
|
-
) {
|
|
27
|
-
lines.push(currentLine);
|
|
28
|
-
currentLine = [];
|
|
29
|
-
}
|
|
30
|
-
currentLine.push(item);
|
|
31
|
-
});
|
|
32
|
-
lines.push(currentLine);
|
|
33
|
-
|
|
34
|
-
lines.forEach((lineItems) => {
|
|
35
|
-
sortByX(lineItems);
|
|
36
|
-
});
|
|
37
|
-
return lines;
|
|
38
|
-
}
|
|
39
|
-
}
|
|
40
|
-
|
|
41
|
-
|
|
1
|
+
/**
|
|
2
|
+
* @description Groups a flat array of `TextItem`s into per-line buckets by
|
|
3
|
+
* clustering items whose y-coordinates differ by less than half of
|
|
4
|
+
* `mostUsedDistance`. Each bucket is then sorted by x so items read left-to-right,
|
|
5
|
+
* which is critical for footnote detection and word ordering in `CompactLines`.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
import TextItem from "./text-item";
|
|
9
|
+
import { sortByX } from "../utils/page-item-functions";
|
|
10
|
+
|
|
11
|
+
// Groups all text items which are on the same y line
|
|
12
|
+
export default class TextItemLineGrouper {
|
|
13
|
+
mostUsedDistance: number;
|
|
14
|
+
|
|
15
|
+
constructor(options: { mostUsedDistance?: number }) {
|
|
16
|
+
this.mostUsedDistance = options.mostUsedDistance || 12;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
group(textItems: TextItem[]): TextItem[][] {
|
|
20
|
+
const lines: TextItem[][] = [];
|
|
21
|
+
var currentLine: TextItem[] = [];
|
|
22
|
+
textItems.forEach((item) => {
|
|
23
|
+
if (
|
|
24
|
+
currentLine.length > 0 &&
|
|
25
|
+
Math.abs(currentLine[0].y - item.y) >= this.mostUsedDistance / 2
|
|
26
|
+
) {
|
|
27
|
+
lines.push(currentLine);
|
|
28
|
+
currentLine = [];
|
|
29
|
+
}
|
|
30
|
+
currentLine.push(item);
|
|
31
|
+
});
|
|
32
|
+
lines.push(currentLine);
|
|
33
|
+
|
|
34
|
+
lines.forEach((lineItems) => {
|
|
35
|
+
sortByX(lineItems);
|
|
36
|
+
});
|
|
37
|
+
return lines;
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
|
package/src/models/word.ts
CHANGED
|
@@ -1,31 +1,31 @@
|
|
|
1
|
-
/**
|
|
2
|
-
* @description Minimal token model for a single word within a `LineItem`.
|
|
3
|
-
* Carries the text `string`, an optional `type` (`WordType`: LINK,
|
|
4
|
-
* FOOTNOTE_LINK, FOOTNOTE) for semantic rendering, and an optional `format`
|
|
5
|
-
* (`WordFormat`: BOLD, OBLIQUE, BOLD_OBLIQUE) for inline HTML styling.
|
|
6
|
-
*/
|
|
7
|
-
|
|
8
|
-
export interface WordFormatEntry {
|
|
9
|
-
name: string;
|
|
10
|
-
startSymbol: string;
|
|
11
|
-
endSymbol: string;
|
|
12
|
-
}
|
|
13
|
-
|
|
14
|
-
export interface WordTypeEntry {
|
|
15
|
-
name: string;
|
|
16
|
-
attachWithoutWhitespace?: boolean;
|
|
17
|
-
plainTextFormat?: boolean;
|
|
18
|
-
toText(string: string): string;
|
|
19
|
-
}
|
|
20
|
-
|
|
21
|
-
export default class Word {
|
|
22
|
-
string: string;
|
|
23
|
-
type: WordTypeEntry | null;
|
|
24
|
-
format: WordFormatEntry | null;
|
|
25
|
-
|
|
26
|
-
constructor(options: { string: string; type?: WordTypeEntry | null; format?: WordFormatEntry | null }) {
|
|
27
|
-
this.string = options.string;
|
|
28
|
-
this.type = options.type ?? null;
|
|
29
|
-
this.format = options.format ?? null;
|
|
30
|
-
}
|
|
31
|
-
}
|
|
1
|
+
/**
|
|
2
|
+
* @description Minimal token model for a single word within a `LineItem`.
|
|
3
|
+
* Carries the text `string`, an optional `type` (`WordType`: LINK,
|
|
4
|
+
* FOOTNOTE_LINK, FOOTNOTE) for semantic rendering, and an optional `format`
|
|
5
|
+
* (`WordFormat`: BOLD, OBLIQUE, BOLD_OBLIQUE) for inline HTML styling.
|
|
6
|
+
*/
|
|
7
|
+
|
|
8
|
+
export interface WordFormatEntry {
|
|
9
|
+
name: string;
|
|
10
|
+
startSymbol: string;
|
|
11
|
+
endSymbol: string;
|
|
12
|
+
}
|
|
13
|
+
|
|
14
|
+
export interface WordTypeEntry {
|
|
15
|
+
name: string;
|
|
16
|
+
attachWithoutWhitespace?: boolean;
|
|
17
|
+
plainTextFormat?: boolean;
|
|
18
|
+
toText(string: string): string;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export default class Word {
|
|
22
|
+
string: string;
|
|
23
|
+
type: WordTypeEntry | null;
|
|
24
|
+
format: WordFormatEntry | null;
|
|
25
|
+
|
|
26
|
+
constructor(options: { string: string; type?: WordTypeEntry | null; format?: WordFormatEntry | null }) {
|
|
27
|
+
this.string = options.string;
|
|
28
|
+
this.type = options.type ?? null;
|
|
29
|
+
this.format = options.format ?? null;
|
|
30
|
+
}
|
|
31
|
+
}
|