@purescraps/purehtml 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (62) hide show
  1. package/.eslintrc.json +21 -0
  2. package/.prettierrc +3 -0
  3. package/README.md +75 -0
  4. package/config.yaml +0 -0
  5. package/dist/examples/simple.d.ts +2 -0
  6. package/dist/examples/simple.js +7586 -0
  7. package/dist/examples/simple.js.map +1 -0
  8. package/dist/src/index.d.ts +24 -0
  9. package/dist/src/index.js +7586 -0
  10. package/dist/src/index.js.map +1 -0
  11. package/examples/simple.html +26 -0
  12. package/examples/simple.ts +12 -0
  13. package/examples/simple.yaml +20 -0
  14. package/jest.config.ts +9 -0
  15. package/package.json +36 -0
  16. package/sample.html +0 -0
  17. package/src/config/config.ts +12 -0
  18. package/src/config/factory.ts +142 -0
  19. package/src/config/index.ts +6 -0
  20. package/src/config/plain-config.ts +19 -0
  21. package/src/config/types/array.test.ts +103 -0
  22. package/src/config/types/array.ts +61 -0
  23. package/src/config/types/constant.ts +15 -0
  24. package/src/config/types/object.test.ts +99 -0
  25. package/src/config/types/object.ts +65 -0
  26. package/src/config/types/primitive.ts +63 -0
  27. package/src/config/types/union.ts +36 -0
  28. package/src/config/types/with-selector.test.ts +32 -0
  29. package/src/config/types/with-selector.ts +26 -0
  30. package/src/config/validator.ts +14 -0
  31. package/src/config-schema.json +144 -0
  32. package/src/config-schema.test.ts +187 -0
  33. package/src/core/argument-types.ts +21 -0
  34. package/src/core/complex-types.ts +4 -0
  35. package/src/core/primitive-types.ts +5 -0
  36. package/src/core/property.ts +3 -0
  37. package/src/core/transformer.ts +21 -0
  38. package/src/errors/invalid-parse-input-error.ts +7 -0
  39. package/src/index.ts +26 -0
  40. package/src/jest.d.ts +5 -0
  41. package/src/setup-jest-matchers.ts +26 -0
  42. package/src/transformers/attr.ts +49 -0
  43. package/src/transformers/exists.ts +20 -0
  44. package/src/transformers/factory.test.ts +17 -0
  45. package/src/transformers/factory.ts +54 -0
  46. package/src/transformers/html.ts +20 -0
  47. package/src/transformers/index.ts +9 -0
  48. package/src/transformers/length.ts +28 -0
  49. package/src/transformers/match-count.ts +20 -0
  50. package/src/transformers/number.ts +30 -0
  51. package/src/transformers/resolve.ts +29 -0
  52. package/src/transformers/transformers.ts +27 -0
  53. package/src/transformers/trim.ts +32 -0
  54. package/src/transformers/url-query-param.ts +65 -0
  55. package/tests/config/constant.test.ts +23 -0
  56. package/tests/extract.test.ts +21 -0
  57. package/tests/fixtures/basic.html +16 -0
  58. package/tests/transformers/attr.test.ts +146 -0
  59. package/tests/transformers/html.test.ts +25 -0
  60. package/tests/transformers/url-query-param.test.ts +85 -0
  61. package/tsconfig.json +10 -0
  62. package/tsup.config.ts +12 -0
@@ -0,0 +1,26 @@
1
+ <!DOCTYPE html>
2
+ <html lang="en">
3
+ <head>
4
+ <meta charset="UTF-8" />
5
+ <meta http-equiv="X-UA-Compatible" content="IE=edge" />
6
+ <meta name="viewport" content="width=device-width, initial-scale=1.0" />
7
+ <title>Sample Document</title>
8
+ </head>
9
+ <body>
10
+ <span class="string">some string</span>
11
+ <span class="number">123</span>
12
+
13
+ <ul class="array">
14
+ <li>
15
+ some string
16
+ </li>
17
+ <li>
18
+ some other string
19
+ </li>
20
+ </ul>
21
+
22
+ <span class="fourth">
23
+ fourth value matches
24
+ </span>
25
+ </body>
26
+ </html>
@@ -0,0 +1,12 @@
1
+ import { load } from 'cheerio';
2
+ import { readFileSync } from 'fs';
3
+ import { extract } from '../src';
4
+ import { ConfigFactory } from '../src/config';
5
+
6
+ const html = readFileSync('examples/simple.html').toString();
7
+ const config = ConfigFactory.fromYAML(
8
+ readFileSync('examples/simple.yaml').toString()
9
+ );
10
+ const $ = load(html);
11
+
12
+ console.log('Simple parse result:', extract($, config, 'https://example.com'));
@@ -0,0 +1,20 @@
1
+ selector: 'body'
2
+
3
+ properties:
4
+ arr:
5
+ selector: ul.array li
6
+ type: array
7
+ transform: trim
8
+ arrMatches:
9
+ selector: ul.array li
10
+ transform: match-count
11
+ exists:
12
+ selector: .non-existent
13
+ transform: exists
14
+ union:
15
+ union:
16
+ - selector: .first
17
+ - selector: .second
18
+ - selector:
19
+ - .third
20
+ - .fourth
package/jest.config.ts ADDED
@@ -0,0 +1,9 @@
1
+ import type { JestConfigWithTsJest } from 'ts-jest';
2
+
3
+ const config: JestConfigWithTsJest = {
4
+ preset: 'ts-jest',
5
+ setupFilesAfterEnv: ['<rootDir>/src/setup-jest-matchers.ts'],
6
+ testEnvironment: 'node',
7
+ };
8
+
9
+ export default config;
package/package.json ADDED
@@ -0,0 +1,36 @@
1
+ {
2
+ "name": "@purescraps/purehtml",
3
+ "version": "0.1.0",
4
+ "description": "",
5
+ "main": "dist/src/index.js",
6
+ "scripts": {
7
+ "build": "tsc --noEmit && tsup",
8
+ "build:fast": "tsup",
9
+ "lint": "eslint .",
10
+ "test": "jest",
11
+ "prepublish": "build"
12
+ },
13
+ "keywords": [],
14
+ "author": "Fatih Kaya <1994274@gmail.com>",
15
+ "license": "MIT",
16
+ "devDependencies": {
17
+ "@types/cheerio": "^0.22.35",
18
+ "@types/jest": "^29.5.12",
19
+ "@types/node": "^20.11.30",
20
+ "@typescript-eslint/eslint-plugin": "^7.3.1",
21
+ "@typescript-eslint/parser": "^7.3.1",
22
+ "ajv": "^8.12.0",
23
+ "eslint": "^8.57.0",
24
+ "jest": "^29.7.0",
25
+ "ts-jest": "^29.1.2",
26
+ "ts-node": "^10.9.2",
27
+ "tsup": "^8.0.2",
28
+ "typescript": "^5.4.3",
29
+ "typescript-eslint": "^7.3.1"
30
+ },
31
+ "dependencies": {
32
+ "cheerio": "^1.0.0-rc.12",
33
+ "yaml": "^2.4.1"
34
+ },
35
+ "packageManager": "pnpm@9.0.6+sha1.648f6014eb363abb36618f2ba59282a9eeb3e879"
36
+ }
package/sample.html ADDED
File without changes
@@ -0,0 +1,12 @@
1
+ import { Property } from '../core/property';
2
+
3
+ export interface ExtractParams {
4
+ $: cheerio.Root;
5
+ $el: cheerio.Cheerio;
6
+ url: string;
7
+ property: Property;
8
+ }
9
+
10
+ export abstract class Config<T = unknown> {
11
+ abstract extract(params: ExtractParams): T;
12
+ }
@@ -0,0 +1,142 @@
1
+ import { parse } from 'yaml';
2
+ import { Transformer } from '../core/transformer';
3
+ import { TransformerFactory } from '../transformers/factory';
4
+ import { Config } from './config';
5
+ import { PlainConfigObject, PlainConfigSelector } from './plain-config';
6
+ import { ArrayConfig } from './types/array';
7
+ import ConstantConfig from './types/constant';
8
+ import { ObjectConfig } from './types/object';
9
+ import { PrimitiveValueConfig } from './types/primitive';
10
+ import UnionConfig from './types/union';
11
+ import ConfigWithSelector from './types/with-selector';
12
+ import { ConfigValidator } from './validator';
13
+
14
+ export class ConfigFactory {
15
+ static fromYAML(yaml: string): Config {
16
+ const plain = parse(yaml);
17
+
18
+ return this.generate(plain);
19
+ }
20
+
21
+ private static generate(plain: PlainConfigObject): Config {
22
+ const validator = new ConfigValidator(plain);
23
+ const errors = validator.validate();
24
+
25
+ if (errors) {
26
+ throw new Error(
27
+ `Invalid config provided. Got validation errors: ${JSON.stringify(
28
+ errors
29
+ )}. Plain config: ${JSON.stringify(plain)}`
30
+ );
31
+ }
32
+
33
+ const {
34
+ constant,
35
+ selector: selectorOrig,
36
+ transform: transformOrig,
37
+ properties,
38
+ items,
39
+ union,
40
+ } = plain;
41
+
42
+ const selector = ConfigFactory.generateSelector(selectorOrig);
43
+
44
+ const expectedType = ConfigFactory.detectExpectedType(plain);
45
+ const transform = transformOrig
46
+ ? ConfigFactory.generateTransform(transformOrig)
47
+ : undefined;
48
+
49
+ switch (expectedType) {
50
+ case 'constant':
51
+ return ConstantConfig.generate(constant);
52
+ case 'object': {
53
+ let propConfigs: ObjectConfig['properties'] | undefined = undefined;
54
+
55
+ if (properties) {
56
+ propConfigs = Object.keys(properties).reduce((acc, key) => {
57
+ acc[key] = this.generate(properties[key]);
58
+
59
+ return acc;
60
+ }, {} as Record<string, Config>);
61
+ }
62
+
63
+ return ObjectConfig.generate(selector, propConfigs);
64
+ }
65
+ case 'array':
66
+ return ArrayConfig.generate(
67
+ selector,
68
+ items && this.generate(items),
69
+ transform
70
+ );
71
+ case 'union':
72
+ return UnionConfig.generate(union!.map((cfg) => this.generate(cfg)));
73
+ default:
74
+ return PrimitiveValueConfig.generate(selector, transform);
75
+ }
76
+ }
77
+
78
+ private static generateSelector(
79
+ selector: PlainConfigSelector
80
+ ): ConfigWithSelector['selector'] {
81
+ if (typeof selector === 'string') {
82
+ return selector || null;
83
+ }
84
+
85
+ if (Array.isArray(selector)) {
86
+ const joined = selector
87
+ .map((s) => ConfigFactory.generateSelector(s))
88
+ .filter((s) => typeof s === 'string')
89
+ .join(', ');
90
+
91
+ return ConfigFactory.generateSelector(joined);
92
+ }
93
+
94
+ if (typeof selector === 'object') {
95
+ const { selector: nselector } = selector;
96
+
97
+ return ConfigFactory.generateSelector(nselector);
98
+ }
99
+
100
+ if (typeof selector === 'undefined') {
101
+ return null;
102
+ }
103
+
104
+ throw new Error(
105
+ `Unexpected selector type: ${typeof selector}. Selector: ${JSON.stringify(
106
+ selector
107
+ )}`
108
+ );
109
+ }
110
+
111
+ private static generateTransform(
112
+ transform: string | string[]
113
+ ): Transformer | Transformer[] {
114
+ if (typeof transform === 'string') {
115
+ return TransformerFactory.create(transform);
116
+ }
117
+
118
+ return transform.map((tr: string) => {
119
+ return TransformerFactory.create(tr);
120
+ });
121
+ }
122
+
123
+ private static detectExpectedType(conf: PlainConfigObject) {
124
+ if ('properties' in conf || conf.type === 'object') {
125
+ return 'object';
126
+ }
127
+
128
+ if ('items' in conf || conf.type === 'array') {
129
+ return 'array';
130
+ }
131
+
132
+ if ('union' in conf) {
133
+ return 'union';
134
+ }
135
+
136
+ if ('constant' in conf) {
137
+ return 'constant';
138
+ }
139
+
140
+ return 'primitive';
141
+ }
142
+ }
@@ -0,0 +1,6 @@
1
+ export * from './types/array';
2
+ export * from './config';
3
+ export * from './factory';
4
+ export * from './types/object';
5
+ export * from './types/primitive';
6
+ export * from './validator';
@@ -0,0 +1,19 @@
1
+ import ConfigWithSelector from './types/with-selector';
2
+
3
+ export type PlainConfigSelector =
4
+ | string
5
+ | string[]
6
+ | { selector: string }
7
+ | undefined;
8
+
9
+ export type PlainConfigObject = {
10
+ selector?: PlainConfigSelector;
11
+ properties?: Record<string, PlainConfigObject>;
12
+ type?: 'string' | 'object' | 'array';
13
+ constant?: unknown;
14
+ transform?: string | string[];
15
+ items?: PlainConfigObject;
16
+ union?: PlainConfigObject[];
17
+ };
18
+
19
+ export type PlainConfig = PlainConfigObject | ConfigWithSelector;
@@ -0,0 +1,103 @@
1
+ import { load } from 'cheerio';
2
+ import { PrimitiveTypes, STRING } from '../../core/primitive-types';
3
+ import { rootProp } from '../../core/property';
4
+ import { TransformParams, Transformer } from '../../core/transformer';
5
+ import { ExtractParams } from '../config';
6
+ import { ArrayConfig } from './array';
7
+ import { PrimitiveValueConfig } from './primitive';
8
+
9
+ const html = `<div id="root">
10
+ <div>foo</div>
11
+ <div>bar</div>
12
+ <div>baz</div>
13
+ </div>`;
14
+
15
+ class FakeTransformer extends Transformer {
16
+ inputType(): PrimitiveTypes {
17
+ return STRING;
18
+ }
19
+
20
+ outputType(): PrimitiveTypes {
21
+ return STRING;
22
+ }
23
+
24
+ transform({ val }: TransformParams): unknown {
25
+ return val;
26
+ }
27
+ }
28
+
29
+ describe('array', () => {
30
+ let params: ExtractParams;
31
+
32
+ beforeAll(() => {
33
+ const $ = load(html);
34
+ const $el = $.root();
35
+
36
+ params = { $, $el, property: rootProp, url: 'https://example.com' };
37
+ });
38
+
39
+ it('should correctly set property', () => {
40
+ const config = ArrayConfig.generate(
41
+ '#root div',
42
+ PrimitiveValueConfig.generate(null)
43
+ );
44
+
45
+ const result = config.extract(params);
46
+
47
+ expect(result).toBeInstanceOf(Array);
48
+ expect(result).toHaveLength(3);
49
+ expect(result).toEqual(['foo', 'bar', 'baz']);
50
+ });
51
+
52
+ it('should respect elementAlreadyMatched', () => {
53
+ const config = ArrayConfig.generate(
54
+ 'invalid-selector',
55
+ PrimitiveValueConfig.generate(null)
56
+ );
57
+
58
+ const result = config.extract({
59
+ ...params,
60
+ $el: params.$('#root div'),
61
+ elementAlreadyMatched: true,
62
+ });
63
+
64
+ expect(result).toBeInstanceOf(Array);
65
+ expect(result).toEqual(['foo', 'bar', 'baz']);
66
+ });
67
+
68
+ describe('transform', () => {
69
+ it('should set "property" correctly', () => {
70
+ const transfomer = new FakeTransformer();
71
+ const spy = jest.spyOn(transfomer, 'transform');
72
+ const config = ArrayConfig.generate(
73
+ '#root div',
74
+ PrimitiveValueConfig.generate(null, transfomer)
75
+ );
76
+
77
+ const property = 'some.nested.prop';
78
+ const result = config.extract({ ...params, property });
79
+
80
+ expect(result).toBeInstanceOf(Array);
81
+ expect(result).toHaveLength(3);
82
+ expect(result).toEqual(['foo', 'bar', 'baz']);
83
+
84
+ const baseParams = { ...params, $el: expect.anything() };
85
+
86
+ expect(spy).toHaveBeenNthCalledWith(1, {
87
+ ...baseParams,
88
+ property: 'some.nested.prop[0]',
89
+ val: 'foo',
90
+ });
91
+ expect(spy).toHaveBeenNthCalledWith(2, {
92
+ ...baseParams,
93
+ property: 'some.nested.prop[1]',
94
+ val: 'bar',
95
+ });
96
+ expect(spy).toHaveBeenNthCalledWith(3, {
97
+ ...baseParams,
98
+ property: 'some.nested.prop[2]',
99
+ val: 'baz',
100
+ });
101
+ });
102
+ });
103
+ });
@@ -0,0 +1,61 @@
1
+ import { Property, rootProp } from '../../core/property';
2
+ import { Config, ExtractParams } from '../config';
3
+ import { PrimitiveValueConfig, Transform } from './primitive';
4
+ import ConfigWithSelector, {
5
+ ConfigWithSelectorExtractParams,
6
+ } from './with-selector';
7
+
8
+ export class ArrayConfig extends ConfigWithSelector {
9
+ static generate(
10
+ selector: ConfigWithSelector['selector'],
11
+ items?: ArrayConfig['items'],
12
+ transform?: Transform
13
+ ): ArrayConfig {
14
+ const conf = new ArrayConfig();
15
+
16
+ conf.selector = selector;
17
+
18
+ if (items) {
19
+ conf.items = items;
20
+ }
21
+
22
+ if (transform) {
23
+ conf.transform = transform;
24
+ }
25
+
26
+ return conf;
27
+ }
28
+
29
+ transform?: Transform;
30
+ items?: Config;
31
+
32
+ private constructor() {
33
+ super();
34
+ }
35
+
36
+ extract(params: ConfigWithSelectorExtractParams) {
37
+ const $el = this.getSelectorMatches(
38
+ params.$el,
39
+ params.elementAlreadyMatched ?? false
40
+ );
41
+ const conf =
42
+ this.items || PrimitiveValueConfig.generate(null, this.transform);
43
+ const extractOpts = { ...params } satisfies ExtractParams;
44
+
45
+ return $el.toArray().map((el, i) => {
46
+ return conf.extract({
47
+ ...extractOpts,
48
+ $el: params.$(el),
49
+ property: this.makeProperty(params.property, i),
50
+ });
51
+ });
52
+ }
53
+
54
+ private makeProperty(property: Property, index: number): Property {
55
+ if (property === rootProp) {
56
+ return `[${index}]`;
57
+ }
58
+
59
+ return `${property}[${index}]`;
60
+ }
61
+ }
@@ -0,0 +1,15 @@
1
+ import { Config } from '../config';
2
+
3
+ export default class ConstantConfig extends Config {
4
+ private constructor(private readonly val: unknown) {
5
+ super();
6
+ }
7
+
8
+ extract() {
9
+ return this.val;
10
+ }
11
+
12
+ static generate(val: unknown) {
13
+ return new ConstantConfig(val);
14
+ }
15
+ }
@@ -0,0 +1,99 @@
1
+ import { load } from 'cheerio';
2
+ import { PrimitiveTypes, STRING } from '../../core/primitive-types';
3
+ import { rootProp } from '../../core/property';
4
+ import { TransformParams, Transformer } from '../../core/transformer';
5
+ import { ExtractParams } from '../config';
6
+ import { ObjectConfig } from './object';
7
+ import { PrimitiveValueConfig } from './primitive';
8
+ import { ArrayConfig } from './array';
9
+ import assert from 'assert';
10
+
11
+ const html = `
12
+ <div id="outside"></div>
13
+ <div id="root">
14
+ <div id="a">foo</div>
15
+ <div id="b">bar</div>
16
+ <div id="c">baz</div>
17
+ </div>`;
18
+
19
+ class FakeTransformer extends Transformer {
20
+ inputType(): PrimitiveTypes {
21
+ return STRING;
22
+ }
23
+
24
+ outputType(): PrimitiveTypes {
25
+ return STRING;
26
+ }
27
+
28
+ transform({ val }: TransformParams): unknown {
29
+ return val;
30
+ }
31
+ }
32
+
33
+ describe('object', () => {
34
+ let params: ExtractParams;
35
+
36
+ beforeAll(() => {
37
+ const $ = load(html);
38
+ const $el = $.root();
39
+
40
+ params = { $, $el, property: rootProp, url: 'https://example.com' };
41
+ });
42
+
43
+ it('should extract object properties', () => {
44
+ const config = ObjectConfig.generate('#root', {
45
+ a: PrimitiveValueConfig.generate('#a'),
46
+ b: PrimitiveValueConfig.generate('#b'),
47
+ c: PrimitiveValueConfig.generate('#c'),
48
+ });
49
+
50
+ const result = config.extract(params);
51
+
52
+ expect(result).toBeInstanceOf(Object);
53
+ expect(result).toEqual({ a: 'foo', b: 'bar', c: 'baz' });
54
+ });
55
+
56
+ it('should correctly set property', () => {
57
+ const transfomer = new FakeTransformer();
58
+ const spy = jest.spyOn(transfomer, 'transform');
59
+ const config = ObjectConfig.generate('#root', {
60
+ a: PrimitiveValueConfig.generate('#a', transfomer),
61
+ b: PrimitiveValueConfig.generate('#b', transfomer),
62
+ c: PrimitiveValueConfig.generate('#c', transfomer),
63
+ });
64
+
65
+ const property = 'some.nested.prop';
66
+ const result = config.extract({ ...params, property });
67
+
68
+ expect(result).toEqual({ a: 'foo', b: 'bar', c: 'baz' });
69
+
70
+ const baseParams = { ...params, $el: expect.anything() };
71
+
72
+ expect(spy).toHaveBeenNthCalledWith(1, {
73
+ ...baseParams,
74
+ property: 'some.nested.prop.a',
75
+ val: 'foo',
76
+ });
77
+ expect(spy).toHaveBeenNthCalledWith(2, {
78
+ ...baseParams,
79
+ property: 'some.nested.prop.b',
80
+ val: 'bar',
81
+ });
82
+ expect(spy).toHaveBeenNthCalledWith(3, {
83
+ ...baseParams,
84
+ property: 'some.nested.prop.c',
85
+ val: 'baz',
86
+ });
87
+ });
88
+
89
+ it('should only match child elements', () => {
90
+ const config = ObjectConfig.generate('#root', {
91
+ divs: ArrayConfig.generate('div'),
92
+ });
93
+
94
+ const result = config.extract(params);
95
+
96
+ assert(result);
97
+ expect(result.divs).toHaveLength(3);
98
+ });
99
+ });
@@ -0,0 +1,65 @@
1
+ import { Property, rootProp } from '../../core/property';
2
+ import { Config } from '../config';
3
+ import ConfigWithSelector, {
4
+ ConfigWithSelectorExtractParams,
5
+ } from './with-selector';
6
+
7
+ export class ObjectConfig extends ConfigWithSelector {
8
+ static generate(
9
+ selector: ConfigWithSelector['selector'],
10
+ properties?: ObjectConfig['properties']
11
+ ): ObjectConfig {
12
+ const conf = new ObjectConfig();
13
+
14
+ conf.selector = selector;
15
+
16
+ if (properties) {
17
+ conf.properties = properties;
18
+ }
19
+
20
+ return conf;
21
+ }
22
+
23
+ properties: {
24
+ [key: string]: Config;
25
+ } = {};
26
+
27
+ private constructor() {
28
+ super();
29
+ }
30
+
31
+ extract(params: ConfigWithSelectorExtractParams) {
32
+ const $parent = params.$el;
33
+ const $el = this.getSelectorMatches(
34
+ $parent,
35
+ params.elementAlreadyMatched ?? false
36
+ );
37
+
38
+ if ($el.length === 0) {
39
+ return null;
40
+ }
41
+
42
+ const props = this.properties;
43
+ const keys = Object.keys(props);
44
+
45
+ return keys.reduce((acc, key) => {
46
+ const config = props[key];
47
+
48
+ acc[key] = config.extract({
49
+ ...params,
50
+ $el,
51
+ property: this.makeProperty(params.property, key),
52
+ });
53
+
54
+ return acc;
55
+ }, {} as Record<string, unknown>);
56
+ }
57
+
58
+ private makeProperty(property: Property, prop: string): Property {
59
+ if (property === rootProp) {
60
+ return prop;
61
+ }
62
+
63
+ return `${property}.${prop}`;
64
+ }
65
+ }