@adguard/dnr-converter 1.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +476 -0
- package/dist/cli.cjs +58912 -0
- package/dist/constants.js +23 -0
- package/dist/declarative-rule/declarative-rule.js +15 -0
- package/dist/declarative-rule/header-info.js +12 -0
- package/dist/declarative-rule/modify-header-info.js +31 -0
- package/dist/declarative-rule/query-key-value.js +12 -0
- package/dist/declarative-rule/query-transform.js +12 -0
- package/dist/declarative-rule/redirect.js +14 -0
- package/dist/declarative-rule/rule-action.js +44 -0
- package/dist/declarative-rule/rule-condition.js +77 -0
- package/dist/declarative-rule/url-transform.js +30 -0
- package/dist/errors/conversion-errors/empty-domains-error.js +24 -0
- package/dist/errors/conversion-errors/empty-resources-error.js +24 -0
- package/dist/errors/conversion-errors/index.js +17 -0
- package/dist/errors/conversion-errors/invalid-declarative-rule-error.js +22 -0
- package/dist/errors/conversion-errors/unsupported-modifier-error.js +20 -0
- package/dist/errors/conversion-errors/unsupported-regexp-error.js +27 -0
- package/dist/errors/converter-options-errors/empty-or-negative-number-of-rules-error.js +18 -0
- package/dist/errors/converter-options-errors/index.js +15 -0
- package/dist/errors/converter-options-errors/negative-number-of-rules-error.js +18 -0
- package/dist/errors/converter-options-errors/resources-path-error.js +18 -0
- package/dist/errors/limitation-errors/index.js +17 -0
- package/dist/errors/limitation-errors/max-scanned-rules-error.js +20 -0
- package/dist/errors/limitation-errors/too-many-error.js +28 -0
- package/dist/errors/limitation-errors/too-many-regexp-rules-error.js +25 -0
- package/dist/errors/limitation-errors/too-many-rules-error.js +25 -0
- package/dist/errors/limitation-errors/too-many-unsafe-rules-error.js +25 -0
- package/dist/errors/unavailable-sources-errors/unavailable-filter-source-error.js +21 -0
- package/dist/errors/unavailable-sources-errors/unavailable-ruleset-source-error.js +21 -0
- package/dist/filter/filter.js +92 -0
- package/dist/filter-converter/filter-converter.js +385 -0
- package/dist/grouped-rules-converters/url-transform-converter.js +340 -0
- package/dist/index.js +30 -0
- package/dist/package.json.js +3 -0
- package/dist/re2-regexp/re2-validator.js +37 -0
- package/dist/re2-regexp/regex-validator-extension.js +30 -0
- package/dist/re2.wasm +0 -0
- package/dist/rule/option-names.js +65 -0
- package/dist/rule/rule-badfilter.js +81 -0
- package/dist/rule/rule-priority.js +88 -0
- package/dist/rule/rule-validator.js +455 -0
- package/dist/rule/rule.js +853 -0
- package/dist/rule/string-utils.js +49 -0
- package/dist/rule-converters/csp-converter.js +105 -0
- package/dist/rule-converters/regular-rule-converter.js +789 -0
- package/dist/rule-converters/remove-header-converter.js +97 -0
- package/dist/rule-converters/rules-converter.js +365 -0
- package/dist/rule-converters/rules-grouper.js +75 -0
- package/dist/rules-scanner/rules-scanner.js +136 -0
- package/dist/ruleset/metadata-rule.js +35 -0
- package/dist/ruleset/metadata-ruleset.js +162 -0
- package/dist/ruleset/rules-hash-map.js +71 -0
- package/dist/ruleset/ruleset-with-source-map.js +406 -0
- package/dist/ruleset/ruleset.js +54 -0
- package/dist/ruleset/source-map.js +283 -0
- package/dist/types/cli/convert-filters.d.ts +32 -0
- package/dist/types/cli/index.d.ts +7 -0
- package/dist/types/cli/utils.d.ts +19 -0
- package/dist/types/src/declarative-rule/declarative-rule.d.ts +32 -0
- package/dist/types/src/declarative-rule/header-info.d.ts +31 -0
- package/dist/types/src/declarative-rule/modify-header-info.d.ts +46 -0
- package/dist/types/src/declarative-rule/query-key-value.d.ts +24 -0
- package/dist/types/src/declarative-rule/query-transform.d.ts +21 -0
- package/dist/types/src/declarative-rule/redirect.d.ts +32 -0
- package/dist/types/src/declarative-rule/rule-action.d.ts +68 -0
- package/dist/types/src/declarative-rule/rule-condition.d.ts +262 -0
- package/dist/types/src/declarative-rule/url-transform.d.ts +69 -0
- package/dist/types/src/errors/conversion-errors/empty-domains-error.d.ts +21 -0
- package/dist/types/src/errors/conversion-errors/empty-resources-error.d.ts +21 -0
- package/dist/types/src/errors/conversion-errors/index.d.ts +27 -0
- package/dist/types/src/errors/conversion-errors/invalid-declarative-rule-error.d.ts +30 -0
- package/dist/types/src/errors/conversion-errors/unsupported-modifier-error.d.ts +20 -0
- package/dist/types/src/errors/conversion-errors/unsupported-regexp-error.d.ts +23 -0
- package/dist/types/src/errors/converter-options-errors/empty-or-negative-number-of-rules-error.d.ts +13 -0
- package/dist/types/src/errors/converter-options-errors/index.d.ts +23 -0
- package/dist/types/src/errors/converter-options-errors/negative-number-of-rules-error.d.ts +13 -0
- package/dist/types/src/errors/converter-options-errors/resources-path-error.d.ts +13 -0
- package/dist/types/src/errors/limitation-errors/index.d.ts +27 -0
- package/dist/types/src/errors/limitation-errors/max-scanned-rules-error.d.ts +18 -0
- package/dist/types/src/errors/limitation-errors/too-many-error.d.ts +33 -0
- package/dist/types/src/errors/limitation-errors/too-many-regexp-rules-error.d.ts +21 -0
- package/dist/types/src/errors/limitation-errors/too-many-rules-error.d.ts +21 -0
- package/dist/types/src/errors/limitation-errors/too-many-unsafe-rules-error.d.ts +21 -0
- package/dist/types/src/errors/unavailable-sources-errors/unavailable-filter-source-error.d.ts +19 -0
- package/dist/types/src/errors/unavailable-sources-errors/unavailable-ruleset-source-error.d.ts +19 -0
- package/dist/types/src/filter/filter.d.ts +74 -0
- package/dist/types/src/filter/types.d.ts +45 -0
- package/dist/types/src/filter-converter/conversion-result.d.ts +35 -0
- package/dist/types/src/filter-converter/filter-converter-options.d.ts +81 -0
- package/dist/types/src/filter-converter/filter-converter.d.ts +197 -0
- package/dist/types/src/index.d.ts +37 -0
- package/dist/types/src/rule/rule-types.d.ts +33 -0
- package/dist/types/src/rule/rule-validator.d.ts +185 -0
- package/dist/types/src/rule/rule.d.ts +269 -0
- package/dist/types/src/ruleset/metadata-ruleset.d.ts +128 -0
- package/dist/types/src/ruleset/rules-hash-map.d.ts +70 -0
- package/dist/types/src/ruleset/ruleset-with-source-map.d.ts +318 -0
- package/dist/types/src/ruleset/ruleset.d.ts +86 -0
- package/dist/types/src/ruleset/source-map.d.ts +45 -0
- package/dist/types/src/ruleset/types.d.ts +68 -0
- package/dist/types/src/utils/is-safe-rule.d.ts +20 -0
- package/dist/types/src/utils/ruleset-utils.d.ts +45 -0
- package/dist/types/src/version.d.ts +6 -0
- package/dist/url-transform-converter.js +338 -0
- package/dist/utils/error.js +22 -0
- package/dist/utils/is-safe-rule.js +25 -0
- package/dist/utils/lazy-loader.js +116 -0
- package/dist/utils/lazy.js +50 -0
- package/dist/utils/ruleset-utils.js +76 -0
- package/dist/utils/string.js +180 -0
- package/dist/utils/valibot.js +74 -0
- package/dist/version.js +7 -0
- package/package.json +122 -0
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { ResourceType } from '../declarative-rule/rule-condition.js';
|
|
2
|
+
import { RuleActionType } from '../declarative-rule/rule-action.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Metadata rule ID. Always the first rule in the rule set.
|
|
6
|
+
*/ const METADATA_RULE_ID = 1;
|
|
7
|
+
/**
|
|
8
|
+
* Metadata key in the rule object.
|
|
9
|
+
*/ const METADATA_KEY = 'metadata';
|
|
10
|
+
/**
|
|
11
|
+
* Dummy rule URL that should not match any request.
|
|
12
|
+
*/ const DUMMY_RULE_URL = 'dummy.rule.adguard.com';
|
|
13
|
+
/**
|
|
14
|
+
* Creates a declarative rule with metadata.
|
|
15
|
+
*
|
|
16
|
+
* @param content Metadata rule configuration.
|
|
17
|
+
*
|
|
18
|
+
* @returns Declarative rule object with metadata.
|
|
19
|
+
*/ const createMetadataRule = (content)=>{
|
|
20
|
+
return {
|
|
21
|
+
id: METADATA_RULE_ID,
|
|
22
|
+
action: {
|
|
23
|
+
type: RuleActionType.Block
|
|
24
|
+
},
|
|
25
|
+
condition: {
|
|
26
|
+
urlFilter: DUMMY_RULE_URL,
|
|
27
|
+
resourceTypes: [
|
|
28
|
+
ResourceType.XmlHttpRequest
|
|
29
|
+
]
|
|
30
|
+
},
|
|
31
|
+
[METADATA_KEY]: content
|
|
32
|
+
};
|
|
33
|
+
};
|
|
34
|
+
|
|
35
|
+
export { createMetadataRule };
|
|
@@ -0,0 +1,162 @@
|
|
|
1
|
+
import * as v from 'valibot';
|
|
2
|
+
import { getRulesetId } from '../utils/ruleset-utils.js';
|
|
3
|
+
import { serializeJson } from '../utils/string.js';
|
|
4
|
+
import { createMetadataRule } from './metadata-rule.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* Metadata ruleset ID.
|
|
8
|
+
*/ const METADATA_RULESET_ID = 0;
|
|
9
|
+
/**
|
|
10
|
+
* Checksum map validator.
|
|
11
|
+
*/ const checksumMapValidator = v.record(v.string(), v.string());
|
|
12
|
+
/**
|
|
13
|
+
* Metadata validator.
|
|
14
|
+
*/ const metadataValidator = v.strictObject({
|
|
15
|
+
/**
|
|
16
|
+
* Checksums for all rulesets.
|
|
17
|
+
*/ checksums: checksumMapValidator,
|
|
18
|
+
/**
|
|
19
|
+
* Additional properties.
|
|
20
|
+
* This field stores any extra information not covered by the other fields.
|
|
21
|
+
* The content of this field is not validated, but it must be JSON serializable.
|
|
22
|
+
* Validation should be performed by users.
|
|
23
|
+
*/ additionalProperties: v.record(v.string(), v.unknown())
|
|
24
|
+
});
|
|
25
|
+
/**
|
|
26
|
+
* Metadata rule validator.
|
|
27
|
+
*
|
|
28
|
+
* Uses `v.looseObject` to allow additional Chrome DNR fields (`id`,
|
|
29
|
+
* `priority`, `action`, `condition` etc.) that are outside our control.
|
|
30
|
+
* The nested `metadata` field uses `v.strictObject()` so unknown metadata
|
|
31
|
+
* keys are rejected at deserialization time.
|
|
32
|
+
*/ const metadataRuleValidator = v.looseObject({
|
|
33
|
+
metadata: metadataValidator
|
|
34
|
+
});
|
|
35
|
+
/**
|
|
36
|
+
* Represents a specialized metadata ruleset for managing and validating
|
|
37
|
+
* metadata associated with various rulesets.
|
|
38
|
+
*
|
|
39
|
+
* This class handles checksums and additional properties, providing methods
|
|
40
|
+
* to manipulate and query this metadata.
|
|
41
|
+
*/ class MetadataRuleset {
|
|
42
|
+
/**
|
|
43
|
+
* Returns rule set id.
|
|
44
|
+
*
|
|
45
|
+
* @returns Rule set id.
|
|
46
|
+
*/ // Note: we prefer `instance.getId()` over `MetadataRuleset.getId(instance)` for consistency.
|
|
47
|
+
// eslint-disable-next-line class-methods-use-this
|
|
48
|
+
getId() {
|
|
49
|
+
return getRulesetId(METADATA_RULESET_ID);
|
|
50
|
+
}
|
|
51
|
+
/**
|
|
52
|
+
* Sets checksum for the specified rule set.
|
|
53
|
+
*
|
|
54
|
+
* @param rulesetId Rule set id.
|
|
55
|
+
* @param checksum Checksum.
|
|
56
|
+
*/ setChecksum(rulesetId, checksum) {
|
|
57
|
+
this.metadataRule.metadata.checksums[rulesetId] = checksum;
|
|
58
|
+
}
|
|
59
|
+
/**
|
|
60
|
+
* Returns checksum for the specified rule set.
|
|
61
|
+
*
|
|
62
|
+
* @param rulesetId Rule set id.
|
|
63
|
+
*
|
|
64
|
+
* @returns Checksum or undefined if not found.
|
|
65
|
+
*/ getChecksum(rulesetId) {
|
|
66
|
+
return this.metadataRule.metadata.checksums[rulesetId];
|
|
67
|
+
}
|
|
68
|
+
/**
|
|
69
|
+
* Returns all rule set ids in the metadata.
|
|
70
|
+
*
|
|
71
|
+
* @returns Rule set ids.
|
|
72
|
+
*/ getRulesetIds() {
|
|
73
|
+
return Object.keys(this.metadataRule.metadata.checksums);
|
|
74
|
+
}
|
|
75
|
+
/**
|
|
76
|
+
* Gets additional property.
|
|
77
|
+
*
|
|
78
|
+
* @param key Property key.
|
|
79
|
+
*
|
|
80
|
+
* @returns Property value or undefined if not found.
|
|
81
|
+
*/ getAdditionalProperty(key) {
|
|
82
|
+
return this.metadataRule.metadata.additionalProperties[key];
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Sets additional property.
|
|
86
|
+
*
|
|
87
|
+
* @param key Property key.
|
|
88
|
+
* @param value Property value. The class does not validate that the value is
|
|
89
|
+
* JSON-serializable; callers are responsible for ensuring serializability.
|
|
90
|
+
*/ setAdditionalProperty(key, value) {
|
|
91
|
+
this.metadataRule.metadata.additionalProperties[key] = value;
|
|
92
|
+
}
|
|
93
|
+
/**
|
|
94
|
+
* Checks whether additional property exists.
|
|
95
|
+
*
|
|
96
|
+
* @param key Property key.
|
|
97
|
+
*
|
|
98
|
+
* @returns Whether the property exists.
|
|
99
|
+
*/ hasAdditionalProperty(key) {
|
|
100
|
+
return Object.hasOwn(this.metadataRule.metadata.additionalProperties, key);
|
|
101
|
+
}
|
|
102
|
+
/**
|
|
103
|
+
* Removes additional property.
|
|
104
|
+
*
|
|
105
|
+
* @param key Property key.
|
|
106
|
+
*/ removeAdditionalProperty(key) {
|
|
107
|
+
delete this.metadataRule.metadata.additionalProperties[key];
|
|
108
|
+
}
|
|
109
|
+
/**
|
|
110
|
+
* Serializes the ruleset to a string.
|
|
111
|
+
*
|
|
112
|
+
* @param pretty Whether to prettify the output.
|
|
113
|
+
*
|
|
114
|
+
* @returns Serialized ruleset.
|
|
115
|
+
*/ serialize(pretty = false) {
|
|
116
|
+
return serializeJson([
|
|
117
|
+
this.metadataRule
|
|
118
|
+
], pretty);
|
|
119
|
+
}
|
|
120
|
+
/**
|
|
121
|
+
* Deserializes the ruleset from a string.
|
|
122
|
+
*
|
|
123
|
+
* @param rawJson Serialized ruleset.
|
|
124
|
+
*
|
|
125
|
+
* @returns Deserialized ruleset.
|
|
126
|
+
*
|
|
127
|
+
* @throws {ValiError} If the metadata fails schema validation.
|
|
128
|
+
* @throws {SyntaxError} If `rawJson` is not valid JSON.
|
|
129
|
+
*/ static deserialize(rawJson) {
|
|
130
|
+
const parsed = JSON.parse(rawJson);
|
|
131
|
+
if (!Array.isArray(parsed) || parsed.length !== 1) {
|
|
132
|
+
throw new Error('Invalid input: expected a single-element array.');
|
|
133
|
+
}
|
|
134
|
+
const { metadata: { checksums, additionalProperties } } = v.parse(metadataRuleValidator, parsed[0]);
|
|
135
|
+
return new MetadataRuleset(checksums, additionalProperties);
|
|
136
|
+
}
|
|
137
|
+
/**
|
|
138
|
+
* Creates an instance of the MetadataRuleset class.
|
|
139
|
+
*
|
|
140
|
+
* @param checksums A map of checksums, where each key corresponds to a
|
|
141
|
+
* rule set ID and each value is the checksum for that ruleset.
|
|
142
|
+
* Defaults to an empty object.
|
|
143
|
+
* @param additionalProperties A collection of additional properties, where
|
|
144
|
+
* keys are property names and values are their associated data. These
|
|
145
|
+
* properties are JSON serializable but not validated by the class.
|
|
146
|
+
* Defaults to an empty object.
|
|
147
|
+
*
|
|
148
|
+
* @note
|
|
149
|
+
* Inputs are shallow-cloned so the instance owns its internal state.
|
|
150
|
+
*/ constructor(checksums = {}, additionalProperties = {}){
|
|
151
|
+
this.metadataRule = createMetadataRule({
|
|
152
|
+
checksums: {
|
|
153
|
+
...checksums
|
|
154
|
+
},
|
|
155
|
+
additionalProperties: {
|
|
156
|
+
...additionalProperties
|
|
157
|
+
}
|
|
158
|
+
});
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
export { METADATA_RULESET_ID, MetadataRuleset };
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* @file Contains the RulesHashMap class for fast rule matching
|
|
3
|
+
* by hash, used for $badfilter checking.
|
|
4
|
+
*/ /**
|
|
5
|
+
* Contains a dictionary where the key is the hash of the rule and the value is
|
|
6
|
+
* a list of sources for the rule. Storing this dictionary is necessary for fast
|
|
7
|
+
* rule matching, which can be negated by $badfilter.
|
|
8
|
+
*/ class RulesHashMap {
|
|
9
|
+
/** @inheritdoc */ findRules(hash) {
|
|
10
|
+
return this.map.get(hash) || [];
|
|
11
|
+
}
|
|
12
|
+
/**
|
|
13
|
+
* Deserializes dictionary from raw string.
|
|
14
|
+
*
|
|
15
|
+
* @param rawString The original dictionary that was serialized into a string.
|
|
16
|
+
*
|
|
17
|
+
* @returns Deserialized dictionary.
|
|
18
|
+
*/ static deserializeSources(rawString) {
|
|
19
|
+
const plainArray = JSON.parse(rawString);
|
|
20
|
+
const allPairs = plainArray.flatMap(([hash, sources])=>{
|
|
21
|
+
return sources.map(([filterId, sourceRuleIndex])=>{
|
|
22
|
+
return {
|
|
23
|
+
hash,
|
|
24
|
+
source: {
|
|
25
|
+
filterId,
|
|
26
|
+
sourceRuleIndex
|
|
27
|
+
}
|
|
28
|
+
};
|
|
29
|
+
});
|
|
30
|
+
});
|
|
31
|
+
return allPairs;
|
|
32
|
+
}
|
|
33
|
+
/** @inheritdoc */ serialize() {
|
|
34
|
+
const arr = Array.from(this.map);
|
|
35
|
+
const serializedValues = arr.map(([hash, source])=>{
|
|
36
|
+
const sources = source.map((s)=>{
|
|
37
|
+
return [
|
|
38
|
+
s.filterId,
|
|
39
|
+
s.sourceRuleIndex
|
|
40
|
+
];
|
|
41
|
+
});
|
|
42
|
+
return [
|
|
43
|
+
hash,
|
|
44
|
+
sources
|
|
45
|
+
];
|
|
46
|
+
});
|
|
47
|
+
return JSON.stringify(serializedValues);
|
|
48
|
+
}
|
|
49
|
+
/**
|
|
50
|
+
* Creates new {@link RulesHashMap}.
|
|
51
|
+
*
|
|
52
|
+
* @param listOfRulesWithHash List of rules hashes and rules sources:
|
|
53
|
+
* filter id with rule index.
|
|
54
|
+
*/ constructor(listOfRulesWithHash){
|
|
55
|
+
/**
|
|
56
|
+
* Map of rule hashes to their sources.
|
|
57
|
+
*/ this.map = new Map();
|
|
58
|
+
listOfRulesWithHash.forEach(({ hash, source })=>{
|
|
59
|
+
const existingValue = this.map.get(hash);
|
|
60
|
+
if (existingValue) {
|
|
61
|
+
existingValue.push(source);
|
|
62
|
+
} else {
|
|
63
|
+
this.map.set(hash, [
|
|
64
|
+
source
|
|
65
|
+
]);
|
|
66
|
+
}
|
|
67
|
+
});
|
|
68
|
+
}
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
export { RulesHashMap };
|
|
@@ -0,0 +1,406 @@
|
|
|
1
|
+
import * as v from 'valibot';
|
|
2
|
+
import { Rule } from '../rule/rule.js';
|
|
3
|
+
import { getErrorMessage } from '../utils/error.js';
|
|
4
|
+
import { LazyLoader } from '../utils/lazy-loader.js';
|
|
5
|
+
import { serializeJson } from '../utils/string.js';
|
|
6
|
+
import { strictObjectByType } from '../utils/valibot.js';
|
|
7
|
+
import { createMetadataRule } from './metadata-rule.js';
|
|
8
|
+
import { SourceMap } from './source-map.js';
|
|
9
|
+
import { UnavailableFilterSourceError } from '../errors/unavailable-sources-errors/unavailable-filter-source-error.js';
|
|
10
|
+
import { UnavailableRulesetSourceError } from '../errors/unavailable-sources-errors/unavailable-ruleset-source-error.js';
|
|
11
|
+
import { DeclarativeRuleValidator } from '../declarative-rule/declarative-rule.js';
|
|
12
|
+
|
|
13
|
+
/**
|
|
14
|
+
* Valibot validator for {@link SerializedRulesetLazyData}.
|
|
15
|
+
*/ const serializedRulesetLazyDataValidator = strictObjectByType({
|
|
16
|
+
sourceMapRaw: v.string(),
|
|
17
|
+
filterIds: v.array(v.number())
|
|
18
|
+
});
|
|
19
|
+
/**
|
|
20
|
+
* Valibot validator for {@link SerializedRulesetData}.
|
|
21
|
+
*/ const serializedRulesetDataValidator = strictObjectByType({
|
|
22
|
+
regexpRulesCount: v.number(),
|
|
23
|
+
unsafeRulesCount: v.number(),
|
|
24
|
+
safeRulesCount: v.number(),
|
|
25
|
+
rulesetHashMapRaw: v.string(),
|
|
26
|
+
badFilterRulesRaw: v.array(v.string()),
|
|
27
|
+
unsafeRules: v.array(DeclarativeRuleValidator)
|
|
28
|
+
});
|
|
29
|
+
/**
|
|
30
|
+
* Keeps converted declarative rules, counters of rules and source map for them.
|
|
31
|
+
*/ class RulesetWithSourceMap {
|
|
32
|
+
/** @inheritdoc */ getUnsafeRules() {
|
|
33
|
+
return Promise.resolve(this.unsafeRules);
|
|
34
|
+
}
|
|
35
|
+
/** @inheritdoc */ getSafeRulesCount() {
|
|
36
|
+
return this.safeRulesCount;
|
|
37
|
+
}
|
|
38
|
+
/** @inheritdoc */ getUnsafeRulesCount() {
|
|
39
|
+
return this.unsafeRulesCount;
|
|
40
|
+
}
|
|
41
|
+
/** @inheritdoc */ getRegexpRulesCount() {
|
|
42
|
+
return this.regexpRulesCount;
|
|
43
|
+
}
|
|
44
|
+
/** @inheritdoc */ getId() {
|
|
45
|
+
return this.id;
|
|
46
|
+
}
|
|
47
|
+
/**
|
|
48
|
+
* Returns a list of pairs of source text rules and their filter identifiers
|
|
49
|
+
* for a given declarative rule identifier.
|
|
50
|
+
*
|
|
51
|
+
* @param declarativeRuleId {@link DeclarativeRule|declarative rule} Id.
|
|
52
|
+
*
|
|
53
|
+
* @returns Promise with list of source rules.
|
|
54
|
+
*
|
|
55
|
+
* @throws An error when filter is not found or filter content is unavailable.
|
|
56
|
+
*/ async findSourceRules(declarativeRuleId) {
|
|
57
|
+
if (!this.sourceMap) {
|
|
58
|
+
return [];
|
|
59
|
+
}
|
|
60
|
+
const sourcePairs = this.sourceMap.getByDeclarativeRuleId(declarativeRuleId);
|
|
61
|
+
const sourceRules = sourcePairs.map(async ({ filterId, sourceRuleIndex })=>{
|
|
62
|
+
const filter = this.filterList.get(filterId);
|
|
63
|
+
if (!filter) {
|
|
64
|
+
throw new Error(`Not found filter list with id: ${filterId}`);
|
|
65
|
+
}
|
|
66
|
+
// `getRuleByIndex` is optional on IFilter — only filters used in
|
|
67
|
+
// the source-map flow are expected to implement it. A filter
|
|
68
|
+
// without it cannot resolve source rules by index, which is a
|
|
69
|
+
// misconfiguration for this flow.
|
|
70
|
+
if (!filter.getRuleByIndex) {
|
|
71
|
+
// eslint-disable-next-line max-len
|
|
72
|
+
const msg = `Filter with id ${filterId} does not implement getRuleByIndex and cannot be used in the source-map flow`;
|
|
73
|
+
throw new UnavailableFilterSourceError(msg, filterId);
|
|
74
|
+
}
|
|
75
|
+
const sourceRule = await filter.getRuleByIndex(sourceRuleIndex);
|
|
76
|
+
return {
|
|
77
|
+
sourceRule,
|
|
78
|
+
filterId
|
|
79
|
+
};
|
|
80
|
+
});
|
|
81
|
+
return Promise.all(sourceRules);
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Run inner lazy deserialization from rule set content provider to load
|
|
85
|
+
* data which is not needed on the creation of rule set:
|
|
86
|
+
* the source map, filter list and declarative rules list.
|
|
87
|
+
*
|
|
88
|
+
* @returns Promise resolving when the content has been loaded.
|
|
89
|
+
*/ loadContent() {
|
|
90
|
+
return this.contentLoader.get();
|
|
91
|
+
}
|
|
92
|
+
/** @inheritdoc */ unloadContent() {
|
|
93
|
+
// Nothing to unload if content was never loaded and no load is pending.
|
|
94
|
+
// Both checks are required: if we only checked `isLoaded()` we would
|
|
95
|
+
// enter the clear branch below while a load is in flight and race
|
|
96
|
+
// with the producer writing to `sourceMap`, `filterList`, and
|
|
97
|
+
// `declarativeRules`.
|
|
98
|
+
if (!this.contentLoader.isLoaded() && !this.contentLoader.isLoading()) {
|
|
99
|
+
return;
|
|
100
|
+
}
|
|
101
|
+
// If a load is in flight, defer the full unload until it settles so
|
|
102
|
+
// we don't race with the producer writing to `sourceMap`, `filterList`,
|
|
103
|
+
// and `declarativeRules`.
|
|
104
|
+
//
|
|
105
|
+
// This defer-during-load dance is specific to `RulesetWithSourceMap`
|
|
106
|
+
// because the producer writes to multiple external fields on `this`.
|
|
107
|
+
// `Filter.unloadContent()` does not need it: `LazyLoader.reset()`
|
|
108
|
+
// already handles defer-during-load for its own cached value, and
|
|
109
|
+
// `Filter`'s only extra state (`ruleByOffset`) is rebuilt lazily on
|
|
110
|
+
// the next `getRuleByIndex()` call, so clearing it is always safe.
|
|
111
|
+
//
|
|
112
|
+
// Fire-and-forget is acceptable because the deferred unload has no
|
|
113
|
+
// observable consumers beyond ruleset state, and any producer error
|
|
114
|
+
// is surfaced to the caller of `loadContent()`.
|
|
115
|
+
if (this.contentLoader.isLoading()) {
|
|
116
|
+
this.contentLoader.get().finally(()=>{
|
|
117
|
+
this.unloadContent();
|
|
118
|
+
}).catch(()=>{
|
|
119
|
+
// Empty on purpose. `.finally()` does not swallow
|
|
120
|
+
// rejections — it re-throws the producer error into this
|
|
121
|
+
// chain. Without `.catch()` that duplicate rejection
|
|
122
|
+
// would bubble up as an unhandled rejection, because
|
|
123
|
+
// nothing awaits this chain. The original error is still
|
|
124
|
+
// delivered to whoever awaited `loadContent()`.
|
|
125
|
+
});
|
|
126
|
+
return;
|
|
127
|
+
}
|
|
128
|
+
// Safely unload all filters in the filter list
|
|
129
|
+
this.filterList.forEach((filter)=>filter.unloadContent());
|
|
130
|
+
// Clear loaded resources
|
|
131
|
+
this.sourceMap = undefined;
|
|
132
|
+
this.declarativeRules = [];
|
|
133
|
+
this.filterList.clear();
|
|
134
|
+
// Mark the content as unloaded
|
|
135
|
+
this.contentLoader.reset();
|
|
136
|
+
}
|
|
137
|
+
/** @inheritdoc */ async getRulesById(declarativeRuleId) {
|
|
138
|
+
try {
|
|
139
|
+
await this.loadContent();
|
|
140
|
+
const originalRules = await this.findSourceRules(declarativeRuleId);
|
|
141
|
+
return originalRules;
|
|
142
|
+
} catch (e) {
|
|
143
|
+
const id = this.getId();
|
|
144
|
+
// eslint-disable-next-line max-len
|
|
145
|
+
const msg = `Cannot extract source rule for given declarativeRuleId ${declarativeRuleId} in rule set '${id}', got error: ${getErrorMessage(e)}`;
|
|
146
|
+
throw new UnavailableRulesetSourceError(msg, id, e);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
/** @inheritdoc */ getBadFilterRules() {
|
|
150
|
+
return this.badFilterRules;
|
|
151
|
+
}
|
|
152
|
+
/** @inheritdoc */ getRulesHashMap() {
|
|
153
|
+
return this.rulesHashMap;
|
|
154
|
+
}
|
|
155
|
+
/** @inheritdoc */ async getDeclarativeRulesIdsBySourceRuleIndex(source) {
|
|
156
|
+
await this.loadContent();
|
|
157
|
+
if (!this.sourceMap) {
|
|
158
|
+
const { filterId, sourceRuleIndex } = source;
|
|
159
|
+
// eslint-disable-next-line max-len
|
|
160
|
+
throw new Error(`Cannot find declarative rules for filter id - ${filterId}, rule index - ${sourceRuleIndex} because source map is undefined in ruleset: ${this.getId()}`);
|
|
161
|
+
}
|
|
162
|
+
return this.sourceMap.getBySourceRuleIndex(source);
|
|
163
|
+
}
|
|
164
|
+
/** @inheritdoc */ async getDeclarativeRules() {
|
|
165
|
+
await this.loadContent();
|
|
166
|
+
return this.declarativeRules;
|
|
167
|
+
}
|
|
168
|
+
/**
|
|
169
|
+
* For provided source rule and filter id return network rule.
|
|
170
|
+
* This method is needed for checking the applicability of $badfilter after
|
|
171
|
+
* a fast-check of rules by comparing only hashes. Afterward, we should
|
|
172
|
+
* build the 'full' Network rule from provided source, not just the hash,
|
|
173
|
+
* to determine the applicability of $badfilter.
|
|
174
|
+
*
|
|
175
|
+
* @param source Source rule and filter id.
|
|
176
|
+
*
|
|
177
|
+
* @returns List of {@link Rule | network rules}.
|
|
178
|
+
*/ static getRuleBySourceRule(source) {
|
|
179
|
+
const { sourceRule, filterId } = source;
|
|
180
|
+
try {
|
|
181
|
+
return Rule.createFromText(filterId, // We don't need line index because this rule
|
|
182
|
+
// will be used only for matching $badfilter rules.
|
|
183
|
+
0, sourceRule);
|
|
184
|
+
} catch (e) {
|
|
185
|
+
return [];
|
|
186
|
+
}
|
|
187
|
+
}
|
|
188
|
+
/**
|
|
189
|
+
* Deserializes rule set to primitives values with lazy load.
|
|
190
|
+
*
|
|
191
|
+
* @param id Id of rule set.
|
|
192
|
+
* @param rawData An item of {@link SerializedRulesetData} for instant
|
|
193
|
+
* creating ruleset. It contains counters for regular declarative and regexp
|
|
194
|
+
* declarative rules, a map of hashes for all rules, and a list of rules
|
|
195
|
+
* with the `$badfilter` modifier.
|
|
196
|
+
* @param loadLazyData An item of {@link SerializedRulesetLazyData} for lazy
|
|
197
|
+
* loading ruleset data to find and display source rules when declarative
|
|
198
|
+
* filtering log is enabled. It includes a map of sources for all rules,
|
|
199
|
+
* a list of declarative rules, and a list of source filter IDs.
|
|
200
|
+
* @param loadDeclarativeRules Loader for ruleset's declarative rules from
|
|
201
|
+
* raw file as a string.
|
|
202
|
+
* @param filterList List of {@link IFilter}.
|
|
203
|
+
*
|
|
204
|
+
* @returns Deserialized rule set.
|
|
205
|
+
*
|
|
206
|
+
* @throws Error {@link UnavailableRulesetSourceError} if rule set source
|
|
207
|
+
* is not available.
|
|
208
|
+
*/ static async deserialize(id, rawData, loadLazyData, loadDeclarativeRules, filterList) {
|
|
209
|
+
let data;
|
|
210
|
+
try {
|
|
211
|
+
const objectFromString = JSON.parse(rawData);
|
|
212
|
+
data = v.parse(serializedRulesetDataValidator, objectFromString);
|
|
213
|
+
} catch (e) {
|
|
214
|
+
// eslint-disable-next-line max-len
|
|
215
|
+
const msg = `Cannot parse serialized ruleset's data with id "${id}", got error: ${getErrorMessage(e)}`;
|
|
216
|
+
throw new UnavailableRulesetSourceError(msg, id, e);
|
|
217
|
+
}
|
|
218
|
+
/**
|
|
219
|
+
* This variable is used as a singleton for all three functions
|
|
220
|
+
* (`loadSourceMap`, `loadFilterList`, `loadDeclarativeRules`) to load
|
|
221
|
+
* data only once.
|
|
222
|
+
*/ let deserializedLazyData;
|
|
223
|
+
const getLazyData = async ()=>{
|
|
224
|
+
if (deserializedLazyData !== undefined) {
|
|
225
|
+
return deserializedLazyData;
|
|
226
|
+
}
|
|
227
|
+
try {
|
|
228
|
+
const lazyData = await loadLazyData();
|
|
229
|
+
const objectFromString = JSON.parse(lazyData);
|
|
230
|
+
const parsed = v.parse(serializedRulesetLazyDataValidator, objectFromString);
|
|
231
|
+
deserializedLazyData = parsed;
|
|
232
|
+
return parsed;
|
|
233
|
+
} catch (e) {
|
|
234
|
+
// eslint-disable-next-line max-len
|
|
235
|
+
const msg = `Cannot parse or load data for lazy metadata for rule set with id "${id}": ${getErrorMessage(e)}`;
|
|
236
|
+
throw new UnavailableRulesetSourceError(msg, id, e);
|
|
237
|
+
}
|
|
238
|
+
};
|
|
239
|
+
const deserialized = {
|
|
240
|
+
id,
|
|
241
|
+
data,
|
|
242
|
+
rulesetContentProvider: {
|
|
243
|
+
loadSourceMap: async ()=>{
|
|
244
|
+
const { sourceMapRaw } = await getLazyData();
|
|
245
|
+
const sources = SourceMap.deserializeSources(sourceMapRaw);
|
|
246
|
+
return new SourceMap(sources);
|
|
247
|
+
},
|
|
248
|
+
loadFilterList: async ()=>{
|
|
249
|
+
const { filterIds } = await getLazyData();
|
|
250
|
+
return filterList.filter((filter)=>filterIds.includes(filter.getId()));
|
|
251
|
+
},
|
|
252
|
+
loadDeclarativeRules: async ()=>{
|
|
253
|
+
const rawFileContent = await loadDeclarativeRules();
|
|
254
|
+
const objectFromString = JSON.parse(rawFileContent);
|
|
255
|
+
const declarativeRules = v.parse(v.array(DeclarativeRuleValidator), objectFromString);
|
|
256
|
+
return declarativeRules;
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
};
|
|
260
|
+
return deserialized;
|
|
261
|
+
}
|
|
262
|
+
/**
|
|
263
|
+
* Helper method to get serialized rule set data.
|
|
264
|
+
*
|
|
265
|
+
* @param unsafeRules List of unsafe rules to add to the serialized output.
|
|
266
|
+
*
|
|
267
|
+
* @returns Serialized rule set data.
|
|
268
|
+
*/ getSerializedRulesetData(unsafeRules) {
|
|
269
|
+
return {
|
|
270
|
+
regexpRulesCount: this.regexpRulesCount,
|
|
271
|
+
unsafeRulesCount: this.unsafeRulesCount,
|
|
272
|
+
// `safeRulesCount` in serialized data represents safe declarative
|
|
273
|
+
// rules only — unsafe rules are tracked in `unsafeRulesCount`
|
|
274
|
+
// and stored in the `unsafeRules` array (metadata).
|
|
275
|
+
safeRulesCount: this.safeRulesCount,
|
|
276
|
+
rulesetHashMapRaw: this.rulesHashMap.serialize(),
|
|
277
|
+
badFilterRulesRaw: this.badFilterRules.map((r)=>r.getText()),
|
|
278
|
+
unsafeRules
|
|
279
|
+
};
|
|
280
|
+
}
|
|
281
|
+
/**
|
|
282
|
+
* Helper method to get serialized rule set lazy data.
|
|
283
|
+
*
|
|
284
|
+
* @returns Serialized rule set lazy data.
|
|
285
|
+
*/ getSerializedRulesetLazyData() {
|
|
286
|
+
return {
|
|
287
|
+
sourceMapRaw: this.sourceMap?.serialize() || '',
|
|
288
|
+
filterIds: Array.from(this.filterList.keys())
|
|
289
|
+
};
|
|
290
|
+
}
|
|
291
|
+
/** @inheritdoc */ async serializeCompact(unsafeRules, prettyPrint = true) {
|
|
292
|
+
try {
|
|
293
|
+
await this.loadContent();
|
|
294
|
+
} catch (e) {
|
|
295
|
+
const id = this.getId();
|
|
296
|
+
// eslint-disable-next-line max-len
|
|
297
|
+
const msg = `Cannot serialize ruleset '${id}' because of not available source, got error: ${getErrorMessage(e)}`;
|
|
298
|
+
throw new UnavailableRulesetSourceError(msg, id, e);
|
|
299
|
+
}
|
|
300
|
+
// TODO: Improve this code once we introduce multiple filters within a single ruleset.
|
|
301
|
+
// Also, do not forget to change metadata rule's structure to store preprocessed
|
|
302
|
+
// filter lists in an array.
|
|
303
|
+
// Currently, we expect that there is only one filter within a single rule set.
|
|
304
|
+
const filter = this.filterList.values().next().value;
|
|
305
|
+
if (!filter) {
|
|
306
|
+
const id = this.getId();
|
|
307
|
+
const msg = `Cannot serialize ruleset '${id}' because of not available filter list`;
|
|
308
|
+
throw new UnavailableRulesetSourceError(msg, id);
|
|
309
|
+
}
|
|
310
|
+
const content = await filter.getContent();
|
|
311
|
+
// Validate the provided unsafe rules against the stored count.
|
|
312
|
+
//
|
|
313
|
+
// Two valid states are intentionally supported:
|
|
314
|
+
// - Non-empty `unsafeRules`: used by the unsafe-rule post-pass (see
|
|
315
|
+
// `@adguard/dnr-rulesets`) which moves unsafe rules out of the
|
|
316
|
+
// declarative list into metadata. Here the length MUST equal
|
|
317
|
+
// `unsafeRulesCount`.
|
|
318
|
+
// - Empty `unsafeRules` (`[]`): used at the initial CLI conversion
|
|
319
|
+
// stage, where no unsafe rules are excluded yet — the count is still
|
|
320
|
+
// recorded in `unsafeRulesCount` for capacity tracking, but the
|
|
321
|
+
// metadata stores an empty set until the post-pass runs.
|
|
322
|
+
//
|
|
323
|
+
// The check therefore only rejects a mismatch when a non-empty array is
|
|
324
|
+
// provided; an empty array is always allowed (initial conversion).
|
|
325
|
+
if (unsafeRules.length > 0 && unsafeRules.length !== this.unsafeRulesCount) {
|
|
326
|
+
const id = this.getId();
|
|
327
|
+
// eslint-disable-next-line max-len
|
|
328
|
+
const msg = `Unsafe rules count is not equal to the length of provided unsafe rules array in rule set '${id}'`;
|
|
329
|
+
throw new Error(msg);
|
|
330
|
+
}
|
|
331
|
+
const metadataRule = createMetadataRule({
|
|
332
|
+
metadata: this.getSerializedRulesetData(unsafeRules),
|
|
333
|
+
lazyMetadata: this.getSerializedRulesetLazyData(),
|
|
334
|
+
filterContent: content
|
|
335
|
+
});
|
|
336
|
+
// Insert metadata rule at the beginning of the rules array without
|
|
337
|
+
// "unshifting" it to avoid mutating the internal state of the Ruleset,
|
|
338
|
+
// which could lead to issues if serializeCompact is called multiple times.
|
|
339
|
+
let declarativeRules = [];
|
|
340
|
+
declarativeRules = declarativeRules.concat(metadataRule);
|
|
341
|
+
const convertedRules = await this.getDeclarativeRules();
|
|
342
|
+
declarativeRules = declarativeRules.concat(convertedRules);
|
|
343
|
+
// Exclude unsafe rules from declarative rules if any are provided.
|
|
344
|
+
if (unsafeRules.length > 0) {
|
|
345
|
+
const unsafeRulesIds = new Set(unsafeRules.map((rule)=>rule.id));
|
|
346
|
+
declarativeRules = declarativeRules.filter((rule)=>{
|
|
347
|
+
return !unsafeRulesIds.has(rule.id);
|
|
348
|
+
});
|
|
349
|
+
}
|
|
350
|
+
const result = serializeJson(declarativeRules, prettyPrint);
|
|
351
|
+
return result;
|
|
352
|
+
}
|
|
353
|
+
/**
|
|
354
|
+
* Constructor of RulesetWithSourceMap.
|
|
355
|
+
*
|
|
356
|
+
* @param id Id of rule set.
|
|
357
|
+
* @param safeRulesCount Number of safe declarative rules
|
|
358
|
+
* (unsafe rules are tracked separately via {@link unsafeRulesCount}).
|
|
359
|
+
* @param unsafeRulesCount Number of unsafe rules.
|
|
360
|
+
* @param regexpRulesCount Number of regexp rules.
|
|
361
|
+
* @param rulesetContentProvider Rule set content provider.
|
|
362
|
+
* @param badFilterRules List of rules with $badfilter modifier.
|
|
363
|
+
* @param rulesHashMap Dictionary with hashes for all source rules.
|
|
364
|
+
* @param unsafeRules List of unsafe DNR rules.
|
|
365
|
+
*/ constructor(id, safeRulesCount, unsafeRulesCount, regexpRulesCount, rulesetContentProvider, badFilterRules, rulesHashMap, unsafeRules){
|
|
366
|
+
/**
|
|
367
|
+
* Array of converted declarative rules.
|
|
368
|
+
*/ this.declarativeRules = [];
|
|
369
|
+
/**
|
|
370
|
+
* Number of safe declarative rules (unsafe rules are stored separately
|
|
371
|
+
* and not counted here).
|
|
372
|
+
*
|
|
373
|
+
* This is needed for the lazy version of the rule set,
|
|
374
|
+
* when content not loaded.
|
|
375
|
+
*/ this.safeRulesCount = 0;
|
|
376
|
+
/**
|
|
377
|
+
* Converted declarative unsafe rules.
|
|
378
|
+
*/ this.unsafeRulesCount = 0;
|
|
379
|
+
/**
|
|
380
|
+
* Converted declarative regexp rules.
|
|
381
|
+
*/ this.regexpRulesCount = 0;
|
|
382
|
+
/**
|
|
383
|
+
* Keeps array of source filter lists.
|
|
384
|
+
*/ this.filterList = new Map();
|
|
385
|
+
this.id = id;
|
|
386
|
+
this.safeRulesCount = safeRulesCount;
|
|
387
|
+
this.unsafeRulesCount = unsafeRulesCount;
|
|
388
|
+
this.regexpRulesCount = regexpRulesCount;
|
|
389
|
+
this.rulesetContentProvider = rulesetContentProvider;
|
|
390
|
+
this.badFilterRules = badFilterRules;
|
|
391
|
+
this.rulesHashMap = rulesHashMap;
|
|
392
|
+
this.unsafeRules = unsafeRules;
|
|
393
|
+
this.contentLoader = new LazyLoader(async ()=>{
|
|
394
|
+
const { loadSourceMap, loadFilterList, loadDeclarativeRules } = this.rulesetContentProvider;
|
|
395
|
+
this.sourceMap = await loadSourceMap();
|
|
396
|
+
this.declarativeRules = await loadDeclarativeRules();
|
|
397
|
+
// TODO: Find a better method to load filters (AG-42364)
|
|
398
|
+
const filtersList = await loadFilterList();
|
|
399
|
+
filtersList.forEach((filter)=>{
|
|
400
|
+
this.filterList.set(filter.getId(), filter);
|
|
401
|
+
});
|
|
402
|
+
});
|
|
403
|
+
}
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
export { RulesetWithSourceMap };
|