@adguard/dnr-converter 1.0.0-alpha.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +476 -0
- package/dist/cli.cjs +58912 -0
- package/dist/constants.js +23 -0
- package/dist/declarative-rule/declarative-rule.js +15 -0
- package/dist/declarative-rule/header-info.js +12 -0
- package/dist/declarative-rule/modify-header-info.js +31 -0
- package/dist/declarative-rule/query-key-value.js +12 -0
- package/dist/declarative-rule/query-transform.js +12 -0
- package/dist/declarative-rule/redirect.js +14 -0
- package/dist/declarative-rule/rule-action.js +44 -0
- package/dist/declarative-rule/rule-condition.js +77 -0
- package/dist/declarative-rule/url-transform.js +30 -0
- package/dist/errors/conversion-errors/empty-domains-error.js +24 -0
- package/dist/errors/conversion-errors/empty-resources-error.js +24 -0
- package/dist/errors/conversion-errors/index.js +17 -0
- package/dist/errors/conversion-errors/invalid-declarative-rule-error.js +22 -0
- package/dist/errors/conversion-errors/unsupported-modifier-error.js +20 -0
- package/dist/errors/conversion-errors/unsupported-regexp-error.js +27 -0
- package/dist/errors/converter-options-errors/empty-or-negative-number-of-rules-error.js +18 -0
- package/dist/errors/converter-options-errors/index.js +15 -0
- package/dist/errors/converter-options-errors/negative-number-of-rules-error.js +18 -0
- package/dist/errors/converter-options-errors/resources-path-error.js +18 -0
- package/dist/errors/limitation-errors/index.js +17 -0
- package/dist/errors/limitation-errors/max-scanned-rules-error.js +20 -0
- package/dist/errors/limitation-errors/too-many-error.js +28 -0
- package/dist/errors/limitation-errors/too-many-regexp-rules-error.js +25 -0
- package/dist/errors/limitation-errors/too-many-rules-error.js +25 -0
- package/dist/errors/limitation-errors/too-many-unsafe-rules-error.js +25 -0
- package/dist/errors/unavailable-sources-errors/unavailable-filter-source-error.js +21 -0
- package/dist/errors/unavailable-sources-errors/unavailable-ruleset-source-error.js +21 -0
- package/dist/filter/filter.js +92 -0
- package/dist/filter-converter/filter-converter.js +385 -0
- package/dist/grouped-rules-converters/url-transform-converter.js +340 -0
- package/dist/index.js +30 -0
- package/dist/package.json.js +3 -0
- package/dist/re2-regexp/re2-validator.js +37 -0
- package/dist/re2-regexp/regex-validator-extension.js +30 -0
- package/dist/re2.wasm +0 -0
- package/dist/rule/option-names.js +65 -0
- package/dist/rule/rule-badfilter.js +81 -0
- package/dist/rule/rule-priority.js +88 -0
- package/dist/rule/rule-validator.js +455 -0
- package/dist/rule/rule.js +853 -0
- package/dist/rule/string-utils.js +49 -0
- package/dist/rule-converters/csp-converter.js +105 -0
- package/dist/rule-converters/regular-rule-converter.js +789 -0
- package/dist/rule-converters/remove-header-converter.js +97 -0
- package/dist/rule-converters/rules-converter.js +365 -0
- package/dist/rule-converters/rules-grouper.js +75 -0
- package/dist/rules-scanner/rules-scanner.js +136 -0
- package/dist/ruleset/metadata-rule.js +35 -0
- package/dist/ruleset/metadata-ruleset.js +162 -0
- package/dist/ruleset/rules-hash-map.js +71 -0
- package/dist/ruleset/ruleset-with-source-map.js +406 -0
- package/dist/ruleset/ruleset.js +54 -0
- package/dist/ruleset/source-map.js +283 -0
- package/dist/types/cli/convert-filters.d.ts +32 -0
- package/dist/types/cli/index.d.ts +7 -0
- package/dist/types/cli/utils.d.ts +19 -0
- package/dist/types/src/declarative-rule/declarative-rule.d.ts +32 -0
- package/dist/types/src/declarative-rule/header-info.d.ts +31 -0
- package/dist/types/src/declarative-rule/modify-header-info.d.ts +46 -0
- package/dist/types/src/declarative-rule/query-key-value.d.ts +24 -0
- package/dist/types/src/declarative-rule/query-transform.d.ts +21 -0
- package/dist/types/src/declarative-rule/redirect.d.ts +32 -0
- package/dist/types/src/declarative-rule/rule-action.d.ts +68 -0
- package/dist/types/src/declarative-rule/rule-condition.d.ts +262 -0
- package/dist/types/src/declarative-rule/url-transform.d.ts +69 -0
- package/dist/types/src/errors/conversion-errors/empty-domains-error.d.ts +21 -0
- package/dist/types/src/errors/conversion-errors/empty-resources-error.d.ts +21 -0
- package/dist/types/src/errors/conversion-errors/index.d.ts +27 -0
- package/dist/types/src/errors/conversion-errors/invalid-declarative-rule-error.d.ts +30 -0
- package/dist/types/src/errors/conversion-errors/unsupported-modifier-error.d.ts +20 -0
- package/dist/types/src/errors/conversion-errors/unsupported-regexp-error.d.ts +23 -0
- package/dist/types/src/errors/converter-options-errors/empty-or-negative-number-of-rules-error.d.ts +13 -0
- package/dist/types/src/errors/converter-options-errors/index.d.ts +23 -0
- package/dist/types/src/errors/converter-options-errors/negative-number-of-rules-error.d.ts +13 -0
- package/dist/types/src/errors/converter-options-errors/resources-path-error.d.ts +13 -0
- package/dist/types/src/errors/limitation-errors/index.d.ts +27 -0
- package/dist/types/src/errors/limitation-errors/max-scanned-rules-error.d.ts +18 -0
- package/dist/types/src/errors/limitation-errors/too-many-error.d.ts +33 -0
- package/dist/types/src/errors/limitation-errors/too-many-regexp-rules-error.d.ts +21 -0
- package/dist/types/src/errors/limitation-errors/too-many-rules-error.d.ts +21 -0
- package/dist/types/src/errors/limitation-errors/too-many-unsafe-rules-error.d.ts +21 -0
- package/dist/types/src/errors/unavailable-sources-errors/unavailable-filter-source-error.d.ts +19 -0
- package/dist/types/src/errors/unavailable-sources-errors/unavailable-ruleset-source-error.d.ts +19 -0
- package/dist/types/src/filter/filter.d.ts +74 -0
- package/dist/types/src/filter/types.d.ts +45 -0
- package/dist/types/src/filter-converter/conversion-result.d.ts +35 -0
- package/dist/types/src/filter-converter/filter-converter-options.d.ts +81 -0
- package/dist/types/src/filter-converter/filter-converter.d.ts +197 -0
- package/dist/types/src/index.d.ts +37 -0
- package/dist/types/src/rule/rule-types.d.ts +33 -0
- package/dist/types/src/rule/rule-validator.d.ts +185 -0
- package/dist/types/src/rule/rule.d.ts +269 -0
- package/dist/types/src/ruleset/metadata-ruleset.d.ts +128 -0
- package/dist/types/src/ruleset/rules-hash-map.d.ts +70 -0
- package/dist/types/src/ruleset/ruleset-with-source-map.d.ts +318 -0
- package/dist/types/src/ruleset/ruleset.d.ts +86 -0
- package/dist/types/src/ruleset/source-map.d.ts +45 -0
- package/dist/types/src/ruleset/types.d.ts +68 -0
- package/dist/types/src/utils/is-safe-rule.d.ts +20 -0
- package/dist/types/src/utils/ruleset-utils.d.ts +45 -0
- package/dist/types/src/version.d.ts +6 -0
- package/dist/url-transform-converter.js +338 -0
- package/dist/utils/error.js +22 -0
- package/dist/utils/is-safe-rule.js +25 -0
- package/dist/utils/lazy-loader.js +116 -0
- package/dist/utils/lazy.js +50 -0
- package/dist/utils/ruleset-utils.js +76 -0
- package/dist/utils/string.js +180 -0
- package/dist/utils/valibot.js +74 -0
- package/dist/version.js +7 -0
- package/package.json +122 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Describes an error when filter source is not available.
|
|
3
|
+
*/ class UnavailableFilterSourceError extends Error {
|
|
4
|
+
/**
|
|
5
|
+
* Describes an error when filter source is not available.
|
|
6
|
+
*
|
|
7
|
+
* @param message Message of error.
|
|
8
|
+
* @param filterId Filter id, the source of which is not available.
|
|
9
|
+
* @param cause Basic error, describes why the source is unavailable.
|
|
10
|
+
*/ constructor(message, filterId, cause){
|
|
11
|
+
super(message, {
|
|
12
|
+
cause
|
|
13
|
+
});
|
|
14
|
+
this.name = this.constructor.name;
|
|
15
|
+
this.filterId = filterId;
|
|
16
|
+
// For proper work of the "instanceof" operator
|
|
17
|
+
Object.setPrototypeOf(this, UnavailableFilterSourceError.prototype);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export { UnavailableFilterSourceError };
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Describes an error when rule set source is not available.
|
|
3
|
+
*/ class UnavailableRulesetSourceError extends Error {
|
|
4
|
+
/**
|
|
5
|
+
* Describes an error when rule set source is not available.
|
|
6
|
+
*
|
|
7
|
+
* @param message Message of error.
|
|
8
|
+
* @param rulesetId Rule set id, the source of which is not available.
|
|
9
|
+
* @param cause Basic error, describes why the source is unavailable.
|
|
10
|
+
*/ constructor(message, rulesetId, cause){
|
|
11
|
+
super(message, {
|
|
12
|
+
cause
|
|
13
|
+
});
|
|
14
|
+
this.name = this.constructor.name;
|
|
15
|
+
this.rulesetId = rulesetId;
|
|
16
|
+
// For proper work of the "instanceof" operator
|
|
17
|
+
Object.setPrototypeOf(this, UnavailableRulesetSourceError.prototype);
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
export { UnavailableRulesetSourceError };
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
import { LazyLoader } from '../utils/lazy-loader.js';
|
|
2
|
+
import { UnavailableFilterSourceError } from '../errors/unavailable-sources-errors/unavailable-filter-source-error.js';
|
|
3
|
+
|
|
4
|
+
/**
|
|
5
|
+
* Concrete filter implementation for the DNR converter.
|
|
6
|
+
*
|
|
7
|
+
* Supports two construction modes:
|
|
8
|
+
* - **Pre-loaded**: `new Filter(id, content)` — content is a plain string,
|
|
9
|
+
* `getContent()` resolves immediately, `unloadContent()` is a no-op.
|
|
10
|
+
* - **Lazy-loaded**: `new Filter(id, source)` — content is fetched on the first
|
|
11
|
+
* call to `getContent()`, with promise deduplication and unload/reload support.
|
|
12
|
+
*
|
|
13
|
+
* Use with {@link FilterConverter}.
|
|
14
|
+
*/ class Filter {
|
|
15
|
+
/** @inheritdoc */ getId() {
|
|
16
|
+
return this.id;
|
|
17
|
+
}
|
|
18
|
+
/** @inheritdoc */ async getContent() {
|
|
19
|
+
if (this.preloadedContent !== null) {
|
|
20
|
+
return this.preloadedContent;
|
|
21
|
+
}
|
|
22
|
+
// contentLoader is non-null whenever preloadedContent is null (constructor invariant).
|
|
23
|
+
return this.contentLoader.get();
|
|
24
|
+
}
|
|
25
|
+
/** @inheritdoc */ async getRuleByIndex(index) {
|
|
26
|
+
const content = await this.getContent();
|
|
27
|
+
// Lazily build an offset->line index on first access so subsequent
|
|
28
|
+
// lookups are O(1). index is node.start from the AGTree AST - the
|
|
29
|
+
// character offset at which a rule's text begins in the raw filter string.
|
|
30
|
+
if (this.ruleByOffset === null) {
|
|
31
|
+
const map = new Map();
|
|
32
|
+
// Use regex that handles both \n and \r\n line endings.
|
|
33
|
+
const lines = content.split(Filter.LINE_SPLITTER);
|
|
34
|
+
let offset = 0;
|
|
35
|
+
for (const line of lines){
|
|
36
|
+
map.set(offset, line);
|
|
37
|
+
// Determine the actual separator length by peeking
|
|
38
|
+
// at the raw content after the line.
|
|
39
|
+
const afterLine = offset + line.length;
|
|
40
|
+
let separatorLength = 0;
|
|
41
|
+
if (afterLine < content.length) {
|
|
42
|
+
separatorLength = content[afterLine] === '\r' ? 2 : 1;
|
|
43
|
+
}
|
|
44
|
+
offset += line.length + separatorLength;
|
|
45
|
+
}
|
|
46
|
+
this.ruleByOffset = map;
|
|
47
|
+
}
|
|
48
|
+
const rule = this.ruleByOffset.get(index);
|
|
49
|
+
if (rule === undefined) {
|
|
50
|
+
throw new Error(`Rule not found at offset ${index} in filter ${this.id}`);
|
|
51
|
+
}
|
|
52
|
+
return rule;
|
|
53
|
+
}
|
|
54
|
+
/** @inheritdoc */ unloadContent() {
|
|
55
|
+
// Pre-loaded filters are not unloadable - no source to reload from.
|
|
56
|
+
// (contentLoader is null iff the filter was constructed with a plain string.)
|
|
57
|
+
if (this.contentLoader === null) {
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
this.contentLoader.reset();
|
|
61
|
+
this.ruleByOffset = null;
|
|
62
|
+
}
|
|
63
|
+
// eslint-disable-next-line jsdoc/require-jsdoc
|
|
64
|
+
constructor(id, contentOrGetSource){
|
|
65
|
+
/**
|
|
66
|
+
* Lazily-built map from rule-start character offset to rule text, used
|
|
67
|
+
* by {@link getRuleByIndex} for O(1) lookups. Rebuilt on first access
|
|
68
|
+
* after content is (re)loaded; cleared on {@link unloadContent}.
|
|
69
|
+
*/ this.ruleByOffset = null;
|
|
70
|
+
this.id = id;
|
|
71
|
+
if (typeof contentOrGetSource === 'string') {
|
|
72
|
+
this.preloadedContent = contentOrGetSource;
|
|
73
|
+
this.contentLoader = null;
|
|
74
|
+
} else {
|
|
75
|
+
this.preloadedContent = null;
|
|
76
|
+
const getSource = contentOrGetSource;
|
|
77
|
+
this.contentLoader = new LazyLoader(async ()=>{
|
|
78
|
+
try {
|
|
79
|
+
return await getSource();
|
|
80
|
+
} catch (e) {
|
|
81
|
+
throw new UnavailableFilterSourceError('Filter content is unavailable', this.id, e);
|
|
82
|
+
}
|
|
83
|
+
});
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
}
|
|
87
|
+
/**
|
|
88
|
+
* Regex that matches both \n and \r\n line endings when splitting
|
|
89
|
+
* filter content into individual rules.
|
|
90
|
+
*/ Filter.LINE_SPLITTER = /\r?\n/;
|
|
91
|
+
|
|
92
|
+
export { Filter };
|
|
@@ -0,0 +1,385 @@
|
|
|
1
|
+
import { EmptyOrNegativeNumberOfRulesError } from '../errors/converter-options-errors/empty-or-negative-number-of-rules-error.js';
|
|
2
|
+
import { NegativeNumberOfRulesError } from '../errors/converter-options-errors/negative-number-of-rules-error.js';
|
|
3
|
+
import { ResourcesPathError } from '../errors/converter-options-errors/resources-path-error.js';
|
|
4
|
+
import { RulesConverter } from '../rule-converters/rules-converter.js';
|
|
5
|
+
import { RulesHashMap } from '../ruleset/rules-hash-map.js';
|
|
6
|
+
import { Ruleset } from '../ruleset/ruleset.js';
|
|
7
|
+
import { RulesetWithSourceMap } from '../ruleset/ruleset-with-source-map.js';
|
|
8
|
+
import { SourceMap } from '../ruleset/source-map.js';
|
|
9
|
+
import { getErrorMessage } from '../utils/error.js';
|
|
10
|
+
import { isSafeRule } from '../utils/is-safe-rule.js';
|
|
11
|
+
import { getRulesetId } from '../utils/ruleset-utils.js';
|
|
12
|
+
import { RulesScanner } from '../rules-scanner/rules-scanner.js';
|
|
13
|
+
|
|
14
|
+
/**
|
|
15
|
+
* Unified filter-to-rules converter. Converts adblock filter lists into
|
|
16
|
+
* declarative rule sets.
|
|
17
|
+
*
|
|
18
|
+
* Supports two conversion modes controlled by the
|
|
19
|
+
* {@link FilterConverterOptions.withSourceMap} flag:
|
|
20
|
+
*
|
|
21
|
+
* - **Simple mode** (`withSourceMap` omitted or `false`): produces
|
|
22
|
+
* {@link IRuleset} with synchronous access to declarative rules.
|
|
23
|
+
*
|
|
24
|
+
* - **Source-map mode** (`withSourceMap: true`): produces
|
|
25
|
+
* {@link IRulesetWithSourceMap} with source maps, hash maps, lazy loading,
|
|
26
|
+
* and full `$badfilter` cross-filter support.
|
|
27
|
+
*/ class FilterConverter {
|
|
28
|
+
/**
|
|
29
|
+
* Returns the rule set ID for a given filter ID.
|
|
30
|
+
*
|
|
31
|
+
* @param filterId Filter ID.
|
|
32
|
+
*
|
|
33
|
+
* @returns Rule set ID string.
|
|
34
|
+
*/ static getRulesetId(filterId) {
|
|
35
|
+
return getRulesetId(filterId);
|
|
36
|
+
}
|
|
37
|
+
/**
|
|
38
|
+
* Converts the provided list of filters into declarative rule sets.
|
|
39
|
+
*
|
|
40
|
+
* By default returns one {@link ConversionResult} per filter. When
|
|
41
|
+
* `options.combine` is `true`, all filters are merged into a single result.
|
|
42
|
+
* When `options.withSourceMap` is `true`, the returned rulesets include
|
|
43
|
+
* source maps, hash maps, and lazy loading.
|
|
44
|
+
*
|
|
45
|
+
* @param filters List of {@link IFilter} to convert.
|
|
46
|
+
* @param options Options from {@link FilterConverterOptions}.
|
|
47
|
+
*
|
|
48
|
+
* @returns Array of {@link ConversionResult} items.
|
|
49
|
+
*
|
|
50
|
+
* @throws Some of {@link ResourcesPathError},
|
|
51
|
+
* {@link EmptyOrNegativeNumberOfRulesError},
|
|
52
|
+
* {@link NegativeNumberOfRulesError}.
|
|
53
|
+
*/ // eslint-disable-next-line class-methods-use-this
|
|
54
|
+
async convert(filters, options) {
|
|
55
|
+
if (options) {
|
|
56
|
+
FilterConverter.checkConverterOptions(options);
|
|
57
|
+
}
|
|
58
|
+
if (options?.withSourceMap) {
|
|
59
|
+
return FilterConverter.convertWithSourceMap(filters, options);
|
|
60
|
+
}
|
|
61
|
+
return FilterConverter.convertSimple(filters, options);
|
|
62
|
+
}
|
|
63
|
+
/**
|
|
64
|
+
* Applies `$badfilter` rules from dynamic rulesets to static rulesets and
|
|
65
|
+
* returns the list of declarative rule IDs that should be disabled.
|
|
66
|
+
*
|
|
67
|
+
* This handles the **dynamic → static** direction of cross-ruleset
|
|
68
|
+
* `$badfilter` application. The reverse direction (static → dynamic) is
|
|
69
|
+
* handled via `options.badFilterRules` during `convert()`.
|
|
70
|
+
*
|
|
71
|
+
* @param dynamicRulesets Converted dynamic rulesets containing potential
|
|
72
|
+
* `$badfilter` rules.
|
|
73
|
+
* @param staticRulesets Already-built static rulesets to match against.
|
|
74
|
+
*
|
|
75
|
+
* @returns Array of {@link UpdateStaticRulesOptions} describing which
|
|
76
|
+
* static ruleset IDs should have which declarative rule IDs disabled.
|
|
77
|
+
*/ // eslint-disable-next-line class-methods-use-this
|
|
78
|
+
async computeRulesToDisable(dynamicRulesets, staticRulesets) {
|
|
79
|
+
const dynamicBadFilterRules = dynamicRulesets.flatMap((rs)=>rs.getBadFilterRules());
|
|
80
|
+
const { declarativeRulesToCancel = [] } = await FilterConverter.collectDeclarativeRulesToCancel(staticRulesets, dynamicBadFilterRules);
|
|
81
|
+
return declarativeRulesToCancel;
|
|
82
|
+
}
|
|
83
|
+
/**
|
|
84
|
+
* Runs the simple conversion flow: scans, converts, wraps in {@link Ruleset}.
|
|
85
|
+
*
|
|
86
|
+
* @param filters Filters to convert.
|
|
87
|
+
* @param options Converter options.
|
|
88
|
+
*
|
|
89
|
+
* @returns Array of simple conversion results.
|
|
90
|
+
*/ static async convertSimple(filters, options) {
|
|
91
|
+
const scannedLimit = options?.maxNumberOfRules ? Math.ceil(options.maxNumberOfRules * FilterConverter.SCANNED_RULES_MULTIPLICATOR) : undefined;
|
|
92
|
+
if (options?.combine) {
|
|
93
|
+
const result = await FilterConverter.convertFiltersSimple(filters, FilterConverter.COMBINED_RULESET_ID, options, scannedLimit);
|
|
94
|
+
return [
|
|
95
|
+
result
|
|
96
|
+
];
|
|
97
|
+
}
|
|
98
|
+
const conversionTasks = filters.map((filter)=>FilterConverter.convertFiltersSimple([
|
|
99
|
+
filter
|
|
100
|
+
], FilterConverter.getRulesetId(filter.getId()), options, scannedLimit));
|
|
101
|
+
return Promise.all(conversionTasks);
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* Scans and converts the given filters into a single simple
|
|
105
|
+
* {@link ConversionResult} identified by `rulesetId`.
|
|
106
|
+
*
|
|
107
|
+
* @param filters Filters to scan and convert.
|
|
108
|
+
* @param rulesetId ID to assign to the produced {@link Ruleset}.
|
|
109
|
+
* @param options Converter options.
|
|
110
|
+
* @param scannedLimit Optional limit on the number of scanned rules.
|
|
111
|
+
*
|
|
112
|
+
* @returns Conversion result for the given filters.
|
|
113
|
+
*/ static async convertFiltersSimple(filters, rulesetId, options, scannedLimit) {
|
|
114
|
+
const { errors: scanErrors, filters: scannedFilters } = await RulesScanner.scanFilters(filters, undefined, scannedLimit);
|
|
115
|
+
const convertedRules = await RulesConverter.convert(scannedFilters, options);
|
|
116
|
+
const { declarativeRules, errors: convertErrors, limitations = [] } = convertedRules;
|
|
117
|
+
const ruleset = new Ruleset(rulesetId, declarativeRules);
|
|
118
|
+
return {
|
|
119
|
+
ruleset,
|
|
120
|
+
errors: scanErrors.concat(convertErrors),
|
|
121
|
+
limitations
|
|
122
|
+
};
|
|
123
|
+
}
|
|
124
|
+
/**
|
|
125
|
+
* Runs the advanced conversion flow: scans with optional `$badfilter`
|
|
126
|
+
* pre-filtering, converts, wraps in {@link RulesetWithSourceMap}.
|
|
127
|
+
*
|
|
128
|
+
* @param filters Filters to convert.
|
|
129
|
+
* @param options Converter options.
|
|
130
|
+
*
|
|
131
|
+
* @returns Array of source-map conversion results.
|
|
132
|
+
*/ static async convertWithSourceMap(filters, options) {
|
|
133
|
+
let skipNegatedRulesFn;
|
|
134
|
+
if (options?.badFilterRules && options.badFilterRules.length > 0) {
|
|
135
|
+
const badFilterHashMap = FilterConverter.buildRulesBadFilterHashMap(options.badFilterRules);
|
|
136
|
+
skipNegatedRulesFn = (r)=>{
|
|
137
|
+
const fastMatched = badFilterHashMap.get(r.hash);
|
|
138
|
+
if (!fastMatched) {
|
|
139
|
+
return true;
|
|
140
|
+
}
|
|
141
|
+
for(let i = 0; i < fastMatched.length; i += 1){
|
|
142
|
+
if (fastMatched[i].negatesBadfilter(r)) {
|
|
143
|
+
return false;
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
return true;
|
|
147
|
+
};
|
|
148
|
+
}
|
|
149
|
+
const scannedLimit = options?.maxNumberOfRules ? Math.ceil(options.maxNumberOfRules * FilterConverter.SCANNED_RULES_MULTIPLICATOR) : undefined;
|
|
150
|
+
if (options?.combine) {
|
|
151
|
+
const { errors: scanErrors, filters: scannedFilters } = await RulesScanner.scanFilters(filters, skipNegatedRulesFn, scannedLimit);
|
|
152
|
+
const convertedRules = await RulesConverter.convert(scannedFilters, options);
|
|
153
|
+
const badFilterRules = scannedFilters.flatMap(({ badFilterRules: rules })=>rules);
|
|
154
|
+
const result = FilterConverter.collectConvertedResult(FilterConverter.COMBINED_RULESET_ID, filters, scannedFilters, convertedRules, badFilterRules);
|
|
155
|
+
result.errors = scanErrors.concat(result.errors);
|
|
156
|
+
return [
|
|
157
|
+
result
|
|
158
|
+
];
|
|
159
|
+
}
|
|
160
|
+
const results = await Promise.all(filters.map(async (filter)=>{
|
|
161
|
+
const { errors: scanErrors, filters: [scannedFilter] } = await RulesScanner.scanFilters([
|
|
162
|
+
filter
|
|
163
|
+
], skipNegatedRulesFn, scannedLimit);
|
|
164
|
+
const convertedRules = await RulesConverter.convert([
|
|
165
|
+
scannedFilter
|
|
166
|
+
], options);
|
|
167
|
+
const result = FilterConverter.collectConvertedResult(FilterConverter.getRulesetId(scannedFilter.id), [
|
|
168
|
+
filter
|
|
169
|
+
], [
|
|
170
|
+
scannedFilter
|
|
171
|
+
], convertedRules, scannedFilter.badFilterRules);
|
|
172
|
+
result.errors = scanErrors.concat(result.errors);
|
|
173
|
+
return result;
|
|
174
|
+
}));
|
|
175
|
+
return results;
|
|
176
|
+
}
|
|
177
|
+
/**
|
|
178
|
+
* Collects {@link ConversionResult} from provided list of raw filters,
|
|
179
|
+
* scanned filters, converted rules and bad filter rules.
|
|
180
|
+
* Creates new {@link RulesetWithSourceMap} and wraps all data for
|
|
181
|
+
* {@link RulesetContentProvider}.
|
|
182
|
+
*
|
|
183
|
+
* @param rulesetId Rule set id.
|
|
184
|
+
* @param filterList List of raw filters.
|
|
185
|
+
* @param scannedFilters Already scanned filters.
|
|
186
|
+
* @param convertedRules Converted rules.
|
|
187
|
+
* @param badFilterRules List of rules with $badfilter modifier.
|
|
188
|
+
*
|
|
189
|
+
* @returns Item of {@link ConversionResult}.
|
|
190
|
+
*/ static collectConvertedResult(rulesetId, filterList, scannedFilters, convertedRules, badFilterRules) {
|
|
191
|
+
const { sourceMapValues, declarativeRules, errors, limitations = [] } = convertedRules;
|
|
192
|
+
const rulesetContent = {
|
|
193
|
+
loadSourceMap: async ()=>new SourceMap(sourceMapValues),
|
|
194
|
+
loadFilterList: async ()=>filterList,
|
|
195
|
+
loadDeclarativeRules: async ()=>declarativeRules
|
|
196
|
+
};
|
|
197
|
+
const listOfRulesWithHash = scannedFilters.flatMap(({ id, rules })=>{
|
|
198
|
+
return rules.map((r)=>({
|
|
199
|
+
hash: r.hash,
|
|
200
|
+
source: {
|
|
201
|
+
sourceRuleIndex: r.index,
|
|
202
|
+
filterId: id
|
|
203
|
+
}
|
|
204
|
+
}));
|
|
205
|
+
});
|
|
206
|
+
const rulesHashMap = new RulesHashMap(listOfRulesWithHash);
|
|
207
|
+
const unsafeRulesCount = declarativeRules.filter((r)=>!isSafeRule(r)).length;
|
|
208
|
+
const safeRulesCount = declarativeRules.length - unsafeRulesCount;
|
|
209
|
+
const regexRulesCount = declarativeRules.filter((r)=>RulesConverter.isRegexRule(r)).length;
|
|
210
|
+
const ruleset = new RulesetWithSourceMap(rulesetId, safeRulesCount, unsafeRulesCount, regexRulesCount, rulesetContent, badFilterRules, rulesHashMap, // Freshly-converted rulesets have no metadata-stored unsafe rules yet;
|
|
211
|
+
// unsafe-rule exclusion is performed later by the serializeCompact() caller.
|
|
212
|
+
[]);
|
|
213
|
+
return {
|
|
214
|
+
ruleset,
|
|
215
|
+
errors,
|
|
216
|
+
limitations
|
|
217
|
+
};
|
|
218
|
+
}
|
|
219
|
+
/**
|
|
220
|
+
* Creates dictionary where key is hash of network rule and value is array
|
|
221
|
+
* of rules with this hash.
|
|
222
|
+
*
|
|
223
|
+
* @param badFilterRules A flat list of `$badfilter` network rules.
|
|
224
|
+
*
|
|
225
|
+
* @returns Dictionary with all $badfilter rules indexed by hash.
|
|
226
|
+
*/ static buildRulesBadFilterHashMap(badFilterRules) {
|
|
227
|
+
const result = new Map();
|
|
228
|
+
badFilterRules.forEach((r)=>{
|
|
229
|
+
const { hash } = r;
|
|
230
|
+
const existing = result.get(hash);
|
|
231
|
+
if (existing) {
|
|
232
|
+
existing.push(r);
|
|
233
|
+
} else {
|
|
234
|
+
result.set(hash, [
|
|
235
|
+
r
|
|
236
|
+
]);
|
|
237
|
+
}
|
|
238
|
+
});
|
|
239
|
+
return result;
|
|
240
|
+
}
|
|
241
|
+
/**
|
|
242
|
+
* Checks if some rules (fastMatchedRulesByHash) from the staticRuleset,
|
|
243
|
+
* which have been fast matched by hash, can be negated with the provided
|
|
244
|
+
* badFilterRule via the `$badfilter` option.
|
|
245
|
+
*
|
|
246
|
+
* @param badFilterRule Network rule with `$badfilter` option.
|
|
247
|
+
* @param staticRuleset Static rule set which contains fast matched rules.
|
|
248
|
+
* @param fastMatchedRulesByHash Rules that have been fast matched by hash
|
|
249
|
+
* for potential negation.
|
|
250
|
+
*
|
|
251
|
+
* @returns List of declarative rule IDs that have been canceled by
|
|
252
|
+
* the provided badFilterRule.
|
|
253
|
+
*/ static async checkFastMatchedRulesCanBeCancelled(badFilterRule, staticRuleset, fastMatchedRulesByHash) {
|
|
254
|
+
const fastMatchedDeclarativeRulesIds = [];
|
|
255
|
+
try {
|
|
256
|
+
const promises = fastMatchedRulesByHash.map(async (source)=>{
|
|
257
|
+
return staticRuleset.getDeclarativeRulesIdsBySourceRuleIndex(source);
|
|
258
|
+
});
|
|
259
|
+
const ids = await Promise.all(promises);
|
|
260
|
+
fastMatchedDeclarativeRulesIds.push(...ids.flat());
|
|
261
|
+
} catch (e) {
|
|
262
|
+
// eslint-disable-next-line max-len
|
|
263
|
+
throw new Error(`Not found declarative rule ids for sources: ${JSON.stringify(fastMatchedRulesByHash)}: ${getErrorMessage(e)}`);
|
|
264
|
+
}
|
|
265
|
+
const disableRuleIds = [];
|
|
266
|
+
for(let i = 0; i < fastMatchedDeclarativeRulesIds.length; i += 1){
|
|
267
|
+
const id = fastMatchedDeclarativeRulesIds[i];
|
|
268
|
+
let matchedSourceRules = [];
|
|
269
|
+
try {
|
|
270
|
+
// eslint-disable-next-line no-await-in-loop
|
|
271
|
+
matchedSourceRules = await staticRuleset.getRulesById(id);
|
|
272
|
+
} catch (e) {
|
|
273
|
+
throw new Error(`Not found sources for declarative rule with id "${id}": ${getErrorMessage(e)}`);
|
|
274
|
+
}
|
|
275
|
+
const rules = matchedSourceRules.flatMap((source)=>{
|
|
276
|
+
return RulesetWithSourceMap.getRuleBySourceRule(source);
|
|
277
|
+
});
|
|
278
|
+
// NOTE: Here we use .some but not .every to simplify first
|
|
279
|
+
// version of applying $badfilter rules.
|
|
280
|
+
const someRulesMatched = rules.flat().some((rule)=>badFilterRule.negatesBadfilter(rule));
|
|
281
|
+
if (someRulesMatched) {
|
|
282
|
+
disableRuleIds.push(id);
|
|
283
|
+
}
|
|
284
|
+
}
|
|
285
|
+
return disableRuleIds;
|
|
286
|
+
}
|
|
287
|
+
/**
|
|
288
|
+
* Applies rules with $badfilter modifier from dynamic rulesets to all rules
|
|
289
|
+
* from static rulesets and returns list of ids of declarative rules to
|
|
290
|
+
* disable them.
|
|
291
|
+
*
|
|
292
|
+
* @param staticRulesets List of converted static rulesets.
|
|
293
|
+
* @param dynamicBadFilterRules List of rules with $badfilter.
|
|
294
|
+
*
|
|
295
|
+
* @returns List of ids of declarative rules to disable them.
|
|
296
|
+
*/ static async collectDeclarativeRulesToCancel(staticRulesets, dynamicBadFilterRules) {
|
|
297
|
+
const declarativeRulesToCancel = [];
|
|
298
|
+
const errors = [];
|
|
299
|
+
for(let i = 0; i < staticRulesets.length; i += 1){
|
|
300
|
+
const staticRuleset = staticRulesets[i];
|
|
301
|
+
const disableRuleIds = [];
|
|
302
|
+
for(let j = 0; j < dynamicBadFilterRules.length; j += 1){
|
|
303
|
+
const badFilterRule = dynamicBadFilterRules[j];
|
|
304
|
+
const hashMap = staticRuleset.getRulesHashMap();
|
|
305
|
+
const fastMatchedRulesByHash = hashMap.findRules(badFilterRule.hash);
|
|
306
|
+
if (fastMatchedRulesByHash.length === 0) {
|
|
307
|
+
continue;
|
|
308
|
+
}
|
|
309
|
+
try {
|
|
310
|
+
// eslint-disable-next-line no-await-in-loop
|
|
311
|
+
const ids = await FilterConverter.checkFastMatchedRulesCanBeCancelled(badFilterRule, staticRuleset, fastMatchedRulesByHash);
|
|
312
|
+
disableRuleIds.push(...ids);
|
|
313
|
+
} catch (e) {
|
|
314
|
+
errors.push(new Error(`Cannot apply badfilter: ${getErrorMessage(e)}`));
|
|
315
|
+
}
|
|
316
|
+
}
|
|
317
|
+
if (disableRuleIds.length > 0) {
|
|
318
|
+
declarativeRulesToCancel.push({
|
|
319
|
+
rulesetId: staticRuleset.getId(),
|
|
320
|
+
disableRuleIds
|
|
321
|
+
});
|
|
322
|
+
}
|
|
323
|
+
}
|
|
324
|
+
return {
|
|
325
|
+
errors,
|
|
326
|
+
declarativeRulesToCancel
|
|
327
|
+
};
|
|
328
|
+
}
|
|
329
|
+
// -------------------------------------------------------------------------
|
|
330
|
+
// Option validation
|
|
331
|
+
// -------------------------------------------------------------------------
|
|
332
|
+
/**
|
|
333
|
+
* Checks that provided converter options are correct.
|
|
334
|
+
*
|
|
335
|
+
* @param options Contains path to web accessible resources,
|
|
336
|
+
* maximum number of converter rules and regexp rules. @see
|
|
337
|
+
* {@link FilterConverterOptions} for details.
|
|
338
|
+
*
|
|
339
|
+
* @throws An {@link ResourcesPathError} if the resources path does not
|
|
340
|
+
* start with a slash or it ends with a slash
|
|
341
|
+
* OR an {@link EmptyOrNegativeNumberOfRulesError} if maximum number of
|
|
342
|
+
* rules is equal or less than 0.
|
|
343
|
+
* OR an {@link NegativeNumberOfRulesError} if maximum number of
|
|
344
|
+
* regexp rules is less than 0.
|
|
345
|
+
*/ static checkConverterOptions(options) {
|
|
346
|
+
const { resourcesPath, maxNumberOfRules, maxNumberOfUnsafeRules, maxNumberOfRegexpRules } = options;
|
|
347
|
+
if (resourcesPath !== undefined) {
|
|
348
|
+
const firstChar = 0;
|
|
349
|
+
const lastChar = resourcesPath.length > 0 ? resourcesPath.length - 1 : 0;
|
|
350
|
+
if (resourcesPath[firstChar] !== '/') {
|
|
351
|
+
const msg = 'Path to web accessible resources should ' + `start with a leading slash: ${resourcesPath}`;
|
|
352
|
+
throw new ResourcesPathError(msg);
|
|
353
|
+
}
|
|
354
|
+
if (resourcesPath[lastChar] === '/') {
|
|
355
|
+
const msg = 'Path to web accessible resources should ' + `not end with a slash: ${resourcesPath}`;
|
|
356
|
+
throw new ResourcesPathError(msg);
|
|
357
|
+
}
|
|
358
|
+
}
|
|
359
|
+
if (maxNumberOfRules !== undefined && maxNumberOfRules <= 0) {
|
|
360
|
+
const msg = 'Maximum number of rules cannot be equal or less than 0';
|
|
361
|
+
throw new EmptyOrNegativeNumberOfRulesError(msg);
|
|
362
|
+
}
|
|
363
|
+
if (maxNumberOfUnsafeRules !== undefined && maxNumberOfUnsafeRules < 0) {
|
|
364
|
+
const msg = 'Maximum number of unsafe rules cannot be less than 0';
|
|
365
|
+
throw new NegativeNumberOfRulesError(msg);
|
|
366
|
+
}
|
|
367
|
+
if (maxNumberOfRegexpRules !== undefined && maxNumberOfRegexpRules < 0) {
|
|
368
|
+
const msg = 'Maximum number of regexp rules cannot be less than 0';
|
|
369
|
+
throw new NegativeNumberOfRulesError(msg);
|
|
370
|
+
}
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
/**
|
|
374
|
+
* Same as chrome.declarativeNetRequest.DYNAMIC_RULESET_ID.
|
|
375
|
+
* Used as the combined ruleset ID when `combine: true`.
|
|
376
|
+
*/ FilterConverter.COMBINED_RULESET_ID = '_dynamic';
|
|
377
|
+
/**
|
|
378
|
+
* Number of scanned rules can be limited via converter options. In this
|
|
379
|
+
* case we increase the limit by 10% to scan more rules in case of some
|
|
380
|
+
* network rules will be combined into one declarative rule. It is safe,
|
|
381
|
+
* because we have double check for maxNumberOfRules on the converted DNR
|
|
382
|
+
* rules.
|
|
383
|
+
*/ FilterConverter.SCANNED_RULES_MULTIPLICATOR = 1.1;
|
|
384
|
+
|
|
385
|
+
export { FilterConverter };
|