@opencraw/core 0.1.2 → 0.1.4

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (195) hide show
  1. package/README.md +1 -1
  2. package/dist/access/access-broker.use-case.d.ts +43 -0
  3. package/dist/access/access-config.error.d.ts +5 -0
  4. package/dist/access/access-config.repository.d.ts +11 -0
  5. package/dist/access/access-json-schema.mapper.d.ts +8 -0
  6. package/dist/access/access-plugin.contract.d.ts +48 -0
  7. package/dist/access/access-preset.store.d.ts +19 -0
  8. package/dist/access/access-profile.contract.d.ts +95 -0
  9. package/dist/access/index.d.ts +11 -0
  10. package/dist/access/render-access.mapper.d.ts +41 -0
  11. package/dist/access/session-id.algorithm.d.ts +8 -0
  12. package/dist/api-steps/extract-from-document.use-case.d.ts +41 -0
  13. package/dist/api-steps/index.d.ts +4 -0
  14. package/dist/api-steps/run-api-step.use-case.d.ts +20 -0
  15. package/dist/api-steps/send-request.use-case.d.ts +23 -0
  16. package/dist/browser-session/browser-profile.store.d.ts +52 -0
  17. package/dist/browser-session/browser-session.config.d.ts +21 -0
  18. package/dist/browser-session/browser.client.d.ts +75 -0
  19. package/dist/browser-session/index.d.ts +6 -0
  20. package/dist/captcha/captcha-budget.model.d.ts +21 -0
  21. package/dist/captcha/captcha-detection.client.d.ts +28 -0
  22. package/dist/captcha/captcha-guard.use-case.d.ts +64 -0
  23. package/dist/captcha/captcha-solver-registry.store.d.ts +19 -0
  24. package/dist/captcha/captcha-solver.contract.d.ts +47 -0
  25. package/dist/captcha/captcha.error.d.ts +13 -0
  26. package/dist/captcha/index.d.ts +10 -0
  27. package/dist/captcha/resolve-captcha.use-case.d.ts +40 -0
  28. package/dist/crawl-events/crawl-event.contract.d.ts +169 -0
  29. package/dist/crawl-events/event-bus.store.d.ts +10 -0
  30. package/dist/crawl-events/index.d.ts +4 -0
  31. package/dist/crawl-events/trace-line.mapper.d.ts +14 -0
  32. package/dist/crawl-execution/bootstrap-session.use-case.d.ts +79 -0
  33. package/dist/crawl-execution/crawl-options.config.d.ts +67 -0
  34. package/dist/crawl-execution/crawl-report.model.d.ts +33 -0
  35. package/dist/crawl-execution/create-crawler.use-case.d.ts +20 -0
  36. package/dist/crawl-execution/index.d.ts +8 -0
  37. package/dist/crawl-execution/rotating-runner.use-case.d.ts +66 -0
  38. package/dist/crawl-execution/run-crawl.use-case.d.ts +17 -0
  39. package/dist/crawl-execution/run-input-recipe.use-case.d.ts +51 -0
  40. package/dist/deck-document/deck-document.model.d.ts +58 -0
  41. package/dist/deck-document/deck-table.algorithm.d.ts +35 -0
  42. package/dist/deck-document/index.d.ts +6 -0
  43. package/dist/deck-document/read-pptx.client.d.ts +16 -0
  44. package/dist/docx-document/index.d.ts +2 -0
  45. package/dist/docx-document/read-docx-html.client.d.ts +23 -0
  46. package/dist/extraction-scope/extraction-scope.model.d.ts +77 -0
  47. package/dist/extraction-scope/index.d.ts +5 -0
  48. package/dist/extraction-scope/live-element.model.d.ts +28 -0
  49. package/dist/hooks/hook-registry.store.d.ts +16 -0
  50. package/dist/hooks/hook.contract.d.ts +17 -0
  51. package/dist/hooks/hook.error.d.ts +7 -0
  52. package/dist/hooks/index.d.ts +4 -0
  53. package/dist/http-session/http-response.contract.d.ts +58 -0
  54. package/dist/http-session/http.client.d.ts +42 -0
  55. package/dist/http-session/index.d.ts +5 -0
  56. package/dist/http-session/text-decoding.algorithm.d.ts +35 -0
  57. package/dist/index.esm.js +1410 -95
  58. package/dist/markdown-document/index.d.ts +3 -0
  59. package/dist/markdown-document/read-markdown.client.d.ts +39 -0
  60. package/dist/output-mapping/coerce-field.mapper.d.ts +19 -0
  61. package/dist/output-mapping/generated-field.mapper.d.ts +16 -0
  62. package/dist/output-mapping/index.d.ts +9 -0
  63. package/dist/output-mapping/map-record.use-case.d.ts +27 -0
  64. package/dist/output-mapping/mapping.error.d.ts +17 -0
  65. package/dist/output-mapping/missing-value.policy.d.ts +14 -0
  66. package/dist/output-mapping/output-field.validator.d.ts +10 -0
  67. package/dist/output-mapping/output-record.model.d.ts +20 -0
  68. package/dist/pdf-document/index.d.ts +7 -0
  69. package/dist/pdf-document/pdf-document.model.d.ts +51 -0
  70. package/dist/pdf-document/pdf-table.algorithm.d.ts +43 -0
  71. package/dist/pdf-document/read-pdf.client.d.ts +18 -0
  72. package/dist/pdf-document/row-assembly.algorithm.d.ts +26 -0
  73. package/dist/recipe-loading/index.d.ts +9 -0
  74. package/dist/recipe-loading/load-recipe-set.use-case.d.ts +45 -0
  75. package/dist/recipe-loading/read-recipe-source.use-case.d.ts +14 -0
  76. package/dist/recipe-loading/recipe-binding.error.d.ts +14 -0
  77. package/dist/recipe-loading/recipe-binding.validator.d.ts +34 -0
  78. package/dist/recipe-loading/recipe-file.repository.d.ts +11 -0
  79. package/dist/recipe-loading/recipe-set.model.d.ts +12 -0
  80. package/dist/recipe-loading/recipe-source.contract.d.ts +28 -0
  81. package/dist/recipe-loading/recipe-text.mapper.d.ts +14 -0
  82. package/dist/recipe-schema/index.d.ts +15 -0
  83. package/dist/recipe-schema/input-recipe.contract.d.ts +156 -0
  84. package/dist/recipe-schema/json-schema.mapper.d.ts +15 -0
  85. package/dist/recipe-schema/output-recipe.contract.d.ts +47 -0
  86. package/dist/recipe-schema/recipe-kind.enum.d.ts +35 -0
  87. package/dist/recipe-schema/recipe-validation.error.d.ts +17 -0
  88. package/dist/recipe-schema/recipe.validator.d.ts +28 -0
  89. package/dist/recipe-schema/step.contract.d.ts +206 -0
  90. package/dist/recipe-schema/transform-rule.contract.d.ts +120 -0
  91. package/dist/record-diff/index.d.ts +4 -0
  92. package/dist/record-diff/record-diff.algorithm.d.ts +82 -0
  93. package/dist/record-diff/records-file.repository.d.ts +10 -0
  94. package/dist/record-sink/dedupe.policy.d.ts +29 -0
  95. package/dist/record-sink/index.d.ts +9 -0
  96. package/dist/record-sink/json-lines-sink.repository.d.ts +19 -0
  97. package/dist/record-sink/memory-sink.repository.d.ts +9 -0
  98. package/dist/record-sink/record-sink.contract.d.ts +17 -0
  99. package/dist/selection/html-selector.algorithm.d.ts +19 -0
  100. package/dist/selection/index.d.ts +8 -0
  101. package/dist/selection/json-path.algorithm.d.ts +9 -0
  102. package/dist/selection/json-text.algorithm.d.ts +58 -0
  103. package/dist/selection/regex.algorithm.d.ts +12 -0
  104. package/dist/selection/take-value.mapper.d.ts +27 -0
  105. package/dist/src/access/access-profile.contract.d.ts +4 -0
  106. package/dist/src/access/index.d.ts +1 -1
  107. package/dist/src/api-steps/extract-from-document.use-case.d.ts +5 -5
  108. package/dist/src/api-steps/send-request.use-case.d.ts +3 -2
  109. package/dist/src/browser-session/browser-profile.store.d.ts +52 -0
  110. package/dist/src/browser-session/browser.client.d.ts +8 -0
  111. package/dist/src/browser-session/index.d.ts +1 -0
  112. package/dist/src/crawl-events/crawl-event.contract.d.ts +8 -0
  113. package/dist/src/crawl-execution/bootstrap-session.use-case.d.ts +23 -2
  114. package/dist/src/crawl-execution/crawl-options.config.d.ts +27 -0
  115. package/dist/src/crawl-execution/rotating-runner.use-case.d.ts +9 -0
  116. package/dist/src/crawl-execution/run-crawl.use-case.d.ts +6 -2
  117. package/dist/src/crawl-execution/run-input-recipe.use-case.d.ts +11 -3
  118. package/dist/src/docx-document/index.d.ts +2 -0
  119. package/dist/src/docx-document/read-docx-html.client.d.ts +23 -0
  120. package/dist/src/extraction-scope/extraction-scope.model.d.ts +2 -1
  121. package/dist/src/http-session/http-response.contract.d.ts +2 -1
  122. package/dist/src/index.d.ts +9 -4
  123. package/dist/src/markdown-document/index.d.ts +1 -1
  124. package/dist/src/markdown-document/read-markdown.client.d.ts +10 -0
  125. package/dist/src/recipe-schema/index.d.ts +2 -2
  126. package/dist/src/recipe-schema/input-recipe.contract.d.ts +24 -1
  127. package/dist/src/recipe-schema/recipe-kind.enum.d.ts +1 -1
  128. package/dist/src/recipe-schema/step.contract.d.ts +4 -0
  129. package/dist/src/record-diff/index.d.ts +4 -0
  130. package/dist/src/record-diff/record-diff.algorithm.d.ts +82 -0
  131. package/dist/src/record-diff/records-file.repository.d.ts +10 -0
  132. package/dist/src/record-sink/dedupe.policy.d.ts +19 -7
  133. package/dist/src/record-sink/index.d.ts +1 -0
  134. package/dist/src/selection/html-selector.algorithm.d.ts +2 -1
  135. package/dist/src/step-flow/for-each.use-case.d.ts +4 -2
  136. package/dist/src/step-flow/host-throttle.policy.d.ts +49 -0
  137. package/dist/src/step-flow/index.d.ts +5 -0
  138. package/dist/src/step-flow/run-gate.policy.d.ts +12 -1
  139. package/dist/src/step-flow/step-runner.contract.d.ts +13 -0
  140. package/dist/src/step-flow/transport-retry.policy.d.ts +76 -0
  141. package/dist/src/web-steps/navigate.use-case.d.ts +3 -2
  142. package/dist/src/web-steps/run-web-step.use-case.d.ts +8 -1
  143. package/dist/src/xml-document/index.d.ts +6 -0
  144. package/dist/src/xml-document/xml-document.model.d.ts +13 -0
  145. package/dist/src/xml-document/xml-parser.client.d.ts +49 -0
  146. package/dist/src/xml-document/xpath.algorithm.d.ts +30 -0
  147. package/dist/step-flow/block-rule.policy.d.ts +21 -0
  148. package/dist/step-flow/blocked.error.d.ts +9 -0
  149. package/dist/step-flow/for-each.use-case.d.ts +20 -0
  150. package/dist/step-flow/host-throttle.policy.d.ts +49 -0
  151. package/dist/step-flow/index.d.ts +15 -0
  152. package/dist/step-flow/paginate.use-case.d.ts +17 -0
  153. package/dist/step-flow/retry.policy.d.ts +21 -0
  154. package/dist/step-flow/run-gate.policy.d.ts +50 -0
  155. package/dist/step-flow/run-steps.use-case.d.ts +39 -0
  156. package/dist/step-flow/step-failure.error.d.ts +14 -0
  157. package/dist/step-flow/step-runner.contract.d.ts +52 -0
  158. package/dist/step-flow/transport-retry.policy.d.ts +76 -0
  159. package/dist/template/expression.algorithm.d.ts +53 -0
  160. package/dist/template/index.d.ts +7 -0
  161. package/dist/template/template.algorithm.d.ts +53 -0
  162. package/dist/template/value-path.algorithm.d.ts +29 -0
  163. package/dist/template/value-text.algorithm.d.ts +18 -0
  164. package/dist/transformation/apply-transform-chain.use-case.d.ts +17 -0
  165. package/dist/transformation/collection.algorithm.d.ts +15 -0
  166. package/dist/transformation/currency.algorithm.d.ts +15 -0
  167. package/dist/transformation/date.algorithm.d.ts +13 -0
  168. package/dist/transformation/index.d.ts +10 -0
  169. package/dist/transformation/lookup.algorithm.d.ts +24 -0
  170. package/dist/transformation/number.algorithm.d.ts +30 -0
  171. package/dist/transformation/string.algorithm.d.ts +18 -0
  172. package/dist/transformation/transform-registry.store.d.ts +36 -0
  173. package/dist/transformation/transform.error.d.ts +8 -0
  174. package/dist/transformation/url.algorithm.d.ts +20 -0
  175. package/dist/web-steps/evaluate-script.use-case.d.ts +10 -0
  176. package/dist/web-steps/extract-from-page.use-case.d.ts +10 -0
  177. package/dist/web-steps/index.d.ts +2 -0
  178. package/dist/web-steps/interact.use-case.d.ts +36 -0
  179. package/dist/web-steps/navigate.use-case.d.ts +15 -0
  180. package/dist/web-steps/run-web-step.use-case.d.ts +37 -0
  181. package/dist/web-steps/snapshot-elements.use-case.d.ts +13 -0
  182. package/dist/workbook-document/csv-parser.algorithm.d.ts +26 -0
  183. package/dist/workbook-document/csv-workbook.mapper.d.ts +24 -0
  184. package/dist/workbook-document/grid-table.algorithm.d.ts +53 -0
  185. package/dist/workbook-document/html-tables.mapper.d.ts +14 -0
  186. package/dist/workbook-document/index.d.ts +9 -0
  187. package/dist/workbook-document/read-xlsx.client.d.ts +18 -0
  188. package/dist/workbook-document/workbook-document.model.d.ts +51 -0
  189. package/dist/xml-document/index.d.ts +6 -0
  190. package/dist/xml-document/xml-document.model.d.ts +13 -0
  191. package/dist/xml-document/xml-parser.client.d.ts +49 -0
  192. package/dist/xml-document/xpath.algorithm.d.ts +30 -0
  193. package/dist/yaml-document/index.d.ts +3 -0
  194. package/dist/yaml-document/read-yaml.client.d.ts +25 -0
  195. package/package.json +4 -2
@@ -0,0 +1,49 @@
1
+ /** A parsed document, as the XPath engine sees it. */
2
+ export type XmlNode = Node;
3
+ /**
4
+ * Parses XML strictly but safely: a malformed document is an error naming
5
+ * the URL and the line; entities a DOCTYPE declares are never expanded (no
6
+ * "billion laughs") and nothing external is ever fetched (no XXE), they stay
7
+ * as they are.
8
+ *
9
+ * @param xml - The document text.
10
+ * @param where - The URL or id, for messages.
11
+ * @returns The document.
12
+ * @throws Error when the text is not well-formed XML.
13
+ */
14
+ export declare function parseXml(xml: string, where: string): Document;
15
+ /**
16
+ * HTML as an XML document, for XPath on fetched pages: parsed the way a
17
+ * browser parses it (a forgiving HTML5 parser that inserts `<tbody>` and
18
+ * closes what the page left open), without namespaces, so `//table/tbody/tr`
19
+ * finds what it finds on the live page.
20
+ *
21
+ * @param html - A page or a fragment.
22
+ * @returns The document.
23
+ */
24
+ export declare function htmlAsXml(html: string): Document;
25
+ /**
26
+ * A copy of a document with every namespace dropped: elements and attributes
27
+ * keep their local names, `xmlns` declarations go. What `ignoreNamespaces`
28
+ * queries run on: `//entry/title` instead of `//atom:entry/atom:title`.
29
+ *
30
+ * @param document - A parsed document.
31
+ * @returns The copy.
32
+ */
33
+ export declare function withoutNamespaces(document: Document): Document;
34
+ /**
35
+ * A node as markup: the element with its children (`outer`), or its children only.
36
+ *
37
+ * @param node - A node.
38
+ * @param outer - Whether to include the node itself.
39
+ * @returns The markup.
40
+ */
41
+ export declare function serializeXml(node: Node, outer: boolean): string;
42
+ /**
43
+ * The text of an XML body, gunzipped first when it is gzip (a `sitemap.xml.gz`).
44
+ *
45
+ * @param bytes - The body.
46
+ * @returns The bytes to decode.
47
+ */
48
+ export declare function gunzipIfNeeded(bytes: Uint8Array): Uint8Array;
49
+ //# sourceMappingURL=xml-parser.client.d.ts.map
@@ -0,0 +1,30 @@
1
+ import type { Take } from '../selection/index.js';
2
+ /** An XPath result: a node, or what a function such as `count()` or `string()` gives. */
3
+ export type XpathValue = Node | string | number | boolean;
4
+ export interface XpathOptions {
5
+ /** Prefix to URI, for queries on namespaced documents (`atom: http://www.w3.org/2005/Atom`). */
6
+ namespaces?: Record<string, string>;
7
+ /** Drop the document's namespaces first, so plain names match (`//entry/title`). */
8
+ ignoreNamespaces?: boolean;
9
+ }
10
+ /**
11
+ * Runs an XPath 1.0 query on a parsed document. Prefixes the root element
12
+ * declares are known without being listed; a default namespace has no
13
+ * prefix, so name one in `namespaces` or use `ignoreNamespaces`.
14
+ *
15
+ * @param document - The document.
16
+ * @param expression - The query.
17
+ * @param options - Namespaces.
18
+ * @returns Every node selected, or the one value a function returned.
19
+ * @throws Error naming the query when it does not parse or uses an unknown prefix.
20
+ */
21
+ export declare function selectXpath(document: Document, expression: string, options?: XpathOptions): XpathValue[];
22
+ /**
23
+ * The value of an XPath result.
24
+ *
25
+ * @param value - A node or a function result.
26
+ * @param take - `text` (whitespace collapsed), `html` (the node's inner markup), `json` (its outer markup), `value` (its text as is), `attr:<name>`.
27
+ * @returns The value; `undefined` for a missing attribute.
28
+ */
29
+ export declare function takeFromXml(value: XpathValue, take: Take): unknown;
30
+ //# sourceMappingURL=xpath.algorithm.d.ts.map
@@ -0,0 +1,3 @@
1
+ export { readYaml } from './read-yaml.client.js';
2
+ export type { YamlRead, YamlScalars } from './read-yaml.client.js';
3
+ //# sourceMappingURL=index.d.ts.map
@@ -0,0 +1,25 @@
1
+ /** How YAML scalars are read: typed as YAML 1.2 types them, or as the text written. */
2
+ export type YamlScalars = 'typed' | 'text';
3
+ /** A YAML text, read. */
4
+ export interface YamlRead {
5
+ /** The document's value; several documents (`---`) give an array of their values. */
6
+ data: unknown;
7
+ documents: number;
8
+ /** What the parser noticed but read anyway: an unknown tag (`!!js/function`) is kept as its plain value. */
9
+ warnings: string[];
10
+ }
11
+ /**
12
+ * Parses YAML with the `yaml` package, imported on first use. The version is
13
+ * pinned to YAML 1.2 (core schema) whatever the document declares: under a
14
+ * `%YAML 1.1` directive, `NO` would read as `false` and `0123` as octal `83`.
15
+ * Merge keys (`<<: *base`) are applied, duplicate keys are an error, aliases
16
+ * are capped, and custom tags never build values: nothing in the text runs.
17
+ *
18
+ * @param text - The YAML.
19
+ * @param source - Where it came from, for messages.
20
+ * @param scalars - `typed` (default), or `text` to keep every scalar as written (`0123` stays `"0123"`).
21
+ * @returns The data, the number of documents and the warnings.
22
+ * @throws Error naming the source, with the line and column, for YAML that does not parse.
23
+ */
24
+ export declare function readYaml(text: string, source: string, scalars?: YamlScalars): Promise<YamlRead>;
25
+ //# sourceMappingURL=read-yaml.client.d.ts.map
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@opencraw/core",
3
- "version": "0.1.2",
3
+ "version": "0.1.4",
4
4
  "type": "module",
5
5
  "main": "./dist/index.esm.js",
6
6
  "module": "./dist/index.esm.js",
@@ -20,13 +20,15 @@
20
20
  "!**/*.js.map"
21
21
  ],
22
22
  "dependencies": {
23
- "@opencraw/office-reader": "^0.0.2",
23
+ "@opencraw/office-reader": "^0.0.3",
24
+ "@xmldom/xmldom": "^0.9.12",
24
25
  "cheerio": "^1.2.0",
25
26
  "domhandler": "^6.0.1",
26
27
  "jsonpath-plus": "^10.4.0",
27
28
  "marked": "^18.0.14",
28
29
  "pdfjs-dist": "^6.3.289",
29
30
  "playwright": "^1.63.0",
31
+ "xpath": "^0.0.34",
30
32
  "yaml": "^2.9.1",
31
33
  "zod": "^4.6.5"
32
34
  },