@tavily/n8n-nodes-tavily 0.2.0 → 0.2.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +4 -0
- package/dist/nodes/Tavily/actions/crawl/crawl.operation.d.ts +33 -1
- package/dist/nodes/Tavily/actions/crawl/crawl.operation.js +35 -127
- package/dist/nodes/Tavily/actions/crawl/crawl.operation.js.map +1 -1
- package/dist/nodes/Tavily/actions/crawl/index.d.ts +2 -2
- package/dist/nodes/Tavily/actions/crawl/index.js +8 -8
- package/dist/nodes/Tavily/actions/crawl/index.js.map +1 -1
- package/dist/nodes/Tavily/actions/crawl/url.operation.d.ts +36 -0
- package/dist/nodes/Tavily/actions/crawl/url.operation.js +58 -0
- package/dist/nodes/Tavily/actions/crawl/url.operation.js.map +1 -0
- package/dist/nodes/Tavily/actions/extract/urls.operation.js +6 -1
- package/dist/nodes/Tavily/actions/extract/urls.operation.js.map +1 -1
- package/dist/nodes/Tavily/actions/node.type.d.ts +1 -0
- package/dist/nodes/Tavily/actions/search/query.operation.js +5 -0
- package/dist/nodes/Tavily/actions/search/query.operation.js.map +1 -1
- package/dist/nodes/Tavily/descriptions/common.descriptions.d.ts +1 -0
- package/dist/nodes/Tavily/descriptions/common.descriptions.js +152 -1
- package/dist/nodes/Tavily/descriptions/common.descriptions.js.map +1 -1
- package/dist/package.json +1 -1
- package/dist/tsconfig.tsbuildinfo +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -80,6 +80,7 @@ Tavily Crawl allows you to intelligently crawl a website from a starting URL to
|
|
|
80
80
|
| Include Image Descriptions | When including images, also add a descriptive text for each image |
|
|
81
81
|
| Include Domains | A list of domains to specifically include in the search results |
|
|
82
82
|
| Exclude Domains | A list of domains to specifically exclude from the search results |
|
|
83
|
+
| Include Favicon | Include the favicon URL for each result |
|
|
83
84
|
|
|
84
85
|
### Tavily Extract Parameters
|
|
85
86
|
|
|
@@ -88,6 +89,8 @@ Tavily Crawl allows you to intelligently crawl a website from a starting URL to
|
|
|
88
89
|
| URLs | One or more URLs to extract content from |
|
|
89
90
|
| Include Images | Include a list of images extracted from each URL |
|
|
90
91
|
| Extract Depth | How deeply to parse each URL (Basic or Advanced) |
|
|
92
|
+
| Format | Format of the extracted web page content (Markdown or Text) |
|
|
93
|
+
| Include Favicon | Include the favicon URL for each result |
|
|
91
94
|
|
|
92
95
|
### Tavily Crawl Parameters
|
|
93
96
|
|
|
@@ -107,6 +110,7 @@ Tavily Crawl allows you to intelligently crawl a website from a starting URL to
|
|
|
107
110
|
| Include Images | Whether to include images in the crawl results |
|
|
108
111
|
| Extract Depth | Extraction depth (Basic or Advanced) |
|
|
109
112
|
| Format | Format of the extracted web page content (Markdown or Text) |
|
|
113
|
+
| Include Favicon | Include the favicon URL for each result |
|
|
110
114
|
|
|
111
115
|
## Troubleshooting
|
|
112
116
|
|
|
@@ -1,4 +1,36 @@
|
|
|
1
1
|
import type { IExecuteFunctions, INodeProperties } from 'n8n-workflow';
|
|
2
2
|
export declare const properties: INodeProperties[];
|
|
3
|
+
export declare const description: {
|
|
4
|
+
displayOptions: {
|
|
5
|
+
hide?: {
|
|
6
|
+
[key: string]: (import("n8n-workflow").NodeParameterValue | import("n8n-workflow").DisplayCondition)[] | undefined;
|
|
7
|
+
} | undefined;
|
|
8
|
+
show?: {
|
|
9
|
+
[key: string]: (import("n8n-workflow").NodeParameterValue | import("n8n-workflow").DisplayCondition)[] | undefined;
|
|
10
|
+
'@version'?: (number | import("n8n-workflow").DisplayCondition)[] | undefined;
|
|
11
|
+
} | undefined;
|
|
12
|
+
hideOnCloud?: boolean | undefined;
|
|
13
|
+
};
|
|
14
|
+
displayName: string;
|
|
15
|
+
name: string;
|
|
16
|
+
type: import("n8n-workflow").NodePropertyTypes;
|
|
17
|
+
typeOptions?: import("n8n-workflow").INodePropertyTypeOptions | undefined;
|
|
18
|
+
default: import("n8n-workflow").NodeParameterValueType;
|
|
19
|
+
description?: string | undefined;
|
|
20
|
+
hint?: string | undefined;
|
|
21
|
+
disabledOptions?: import("n8n-workflow").IDisplayOptions | undefined;
|
|
22
|
+
options?: (INodeProperties | import("n8n-workflow").INodePropertyOptions | import("n8n-workflow").INodePropertyCollection)[] | undefined;
|
|
23
|
+
placeholder?: string | undefined;
|
|
24
|
+
isNodeSetting?: boolean | undefined;
|
|
25
|
+
noDataExpression?: boolean | undefined;
|
|
26
|
+
required?: boolean | undefined;
|
|
27
|
+
routing?: import("n8n-workflow").INodePropertyRouting | undefined;
|
|
28
|
+
credentialTypes?: ("extends:oAuth2Api" | "extends:oAuth1Api" | "has:authenticate" | "has:genericAuth")[] | undefined;
|
|
29
|
+
extractValue?: import("n8n-workflow").INodePropertyValueExtractorRegex | undefined;
|
|
30
|
+
modes?: import("n8n-workflow").INodePropertyMode[] | undefined;
|
|
31
|
+
requiresDataPath?: "single" | "multiple" | undefined;
|
|
32
|
+
doNotInherit?: boolean | undefined;
|
|
33
|
+
validateType?: keyof import("n8n-workflow").FieldTypeMap | undefined;
|
|
34
|
+
ignoreValidationDuringExecution?: boolean | undefined;
|
|
35
|
+
}[];
|
|
3
36
|
export declare function execute(this: IExecuteFunctions, index: number): Promise<import("n8n-workflow").INodeExecutionData[]>;
|
|
4
|
-
export declare const description: INodeProperties[];
|
|
@@ -1,7 +1,9 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.
|
|
3
|
+
exports.execute = exports.description = exports.properties = void 0;
|
|
4
4
|
const transport_1 = require("../../transport");
|
|
5
|
+
const common_descriptions_1 = require("../../descriptions/common.descriptions");
|
|
6
|
+
const display_1 = require("../../display");
|
|
5
7
|
exports.properties = [
|
|
6
8
|
{
|
|
7
9
|
displayName: 'URL',
|
|
@@ -10,137 +12,43 @@ exports.properties = [
|
|
|
10
12
|
required: true,
|
|
11
13
|
default: '',
|
|
12
14
|
description: 'The root URL to begin the crawl',
|
|
13
|
-
placeholder: 'https://www.example.com'
|
|
15
|
+
placeholder: 'https://www.example.com',
|
|
16
|
+
displayOptions: {
|
|
17
|
+
show: {
|
|
18
|
+
resource: ['crawl'],
|
|
19
|
+
},
|
|
20
|
+
},
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
displayName: 'Options',
|
|
24
|
+
name: 'options',
|
|
25
|
+
type: 'collection',
|
|
26
|
+
placeholder: 'Add option',
|
|
27
|
+
default: {},
|
|
28
|
+
options: common_descriptions_1.crawlOptions.filter(option => option.name !== 'url'),
|
|
29
|
+
displayOptions: {
|
|
30
|
+
show: {
|
|
31
|
+
resource: ['crawl'],
|
|
32
|
+
},
|
|
33
|
+
},
|
|
14
34
|
},
|
|
15
|
-
{
|
|
16
|
-
displayName: 'Instructions',
|
|
17
|
-
name: 'instructions',
|
|
18
|
-
type: 'string',
|
|
19
|
-
default: '',
|
|
20
|
-
description: 'Natural language instructions for the crawler'
|
|
21
|
-
},
|
|
22
|
-
{
|
|
23
|
-
displayName: 'Max Depth',
|
|
24
|
-
name: 'max_depth',
|
|
25
|
-
type: 'number',
|
|
26
|
-
default: 1,
|
|
27
|
-
description: 'Max depth of the crawl',
|
|
28
|
-
typeOptions: { minValue: 1 }
|
|
29
|
-
},
|
|
30
|
-
{
|
|
31
|
-
displayName: 'Max Breadth',
|
|
32
|
-
name: 'max_breadth',
|
|
33
|
-
type: 'number',
|
|
34
|
-
default: 20,
|
|
35
|
-
description: 'Max number of links to follow per level',
|
|
36
|
-
typeOptions: { minValue: 1 }
|
|
37
|
-
},
|
|
38
|
-
{
|
|
39
|
-
displayName: 'Limit',
|
|
40
|
-
name: 'limit',
|
|
41
|
-
type: 'number',
|
|
42
|
-
default: 50,
|
|
43
|
-
description: 'Max number of results to return',
|
|
44
|
-
typeOptions: { minValue: 1 }
|
|
45
|
-
},
|
|
46
|
-
{
|
|
47
|
-
displayName: 'Categories',
|
|
48
|
-
name: 'categories',
|
|
49
|
-
type: 'multiOptions',
|
|
50
|
-
default: [],
|
|
51
|
-
options: [
|
|
52
|
-
{ name: 'About', value: 'About' },
|
|
53
|
-
{ name: 'Blog', value: 'Blog' },
|
|
54
|
-
{ name: 'Careers', value: 'Careers' },
|
|
55
|
-
{ name: 'Community', value: 'Community' },
|
|
56
|
-
{ name: 'Contact', value: 'Contact' },
|
|
57
|
-
{ name: 'Developers', value: 'Developers' },
|
|
58
|
-
{ name: 'Documentation', value: 'Documentation' },
|
|
59
|
-
{ name: 'Media', value: 'Media' },
|
|
60
|
-
{ name: 'Pricing', value: 'Pricing' },
|
|
61
|
-
],
|
|
62
|
-
description: 'Filter URLs using predefined categories',
|
|
63
|
-
},
|
|
64
|
-
{
|
|
65
|
-
displayName: 'Select Paths',
|
|
66
|
-
name: 'select_paths',
|
|
67
|
-
type: 'string',
|
|
68
|
-
typeOptions: { multipleValues: true },
|
|
69
|
-
default: [],
|
|
70
|
-
description: 'Regex patterns to select only URLs with specific path patterns'
|
|
71
|
-
},
|
|
72
|
-
{
|
|
73
|
-
displayName: 'Select Domains',
|
|
74
|
-
name: 'select_domains',
|
|
75
|
-
type: 'string',
|
|
76
|
-
typeOptions: { multipleValues: true },
|
|
77
|
-
default: [],
|
|
78
|
-
description: 'Regex patterns to select crawling to specific domains or subdomains'
|
|
79
|
-
},
|
|
80
|
-
{
|
|
81
|
-
displayName: 'Exclude Paths',
|
|
82
|
-
name: 'exclude_paths',
|
|
83
|
-
type: 'string',
|
|
84
|
-
typeOptions: { multipleValues: true },
|
|
85
|
-
default: [],
|
|
86
|
-
description: 'Regex patterns to exclude URLs with specific path patterns'
|
|
87
|
-
},
|
|
88
|
-
{
|
|
89
|
-
displayName: 'Exclude Domains',
|
|
90
|
-
name: 'exclude_domains',
|
|
91
|
-
type: 'string',
|
|
92
|
-
typeOptions: { multipleValues: true },
|
|
93
|
-
default: [],
|
|
94
|
-
description: 'Regex patterns to exclude specific domains or subdomains from crawling'
|
|
95
|
-
},
|
|
96
|
-
{
|
|
97
|
-
displayName: 'Allow External',
|
|
98
|
-
name: 'allow_external',
|
|
99
|
-
type: 'boolean',
|
|
100
|
-
default: false,
|
|
101
|
-
description: 'Whether to allow following links that go to external domains'
|
|
102
|
-
},
|
|
103
|
-
{
|
|
104
|
-
displayName: 'Include Images',
|
|
105
|
-
name: 'include_images',
|
|
106
|
-
type: 'boolean',
|
|
107
|
-
default: false,
|
|
108
|
-
description: 'Whether to include images in the crawl results'
|
|
109
|
-
},
|
|
110
|
-
{
|
|
111
|
-
displayName: 'Extract Depth',
|
|
112
|
-
name: 'extract_depth',
|
|
113
|
-
type: 'options',
|
|
114
|
-
default: 'basic',
|
|
115
|
-
options: [
|
|
116
|
-
{ name: 'Basic', value: 'basic' },
|
|
117
|
-
{ name: 'Advanced', value: 'advanced' }
|
|
118
|
-
],
|
|
119
|
-
description: 'Extraction depth'
|
|
120
|
-
},
|
|
121
|
-
{
|
|
122
|
-
displayName: 'Format',
|
|
123
|
-
name: 'format',
|
|
124
|
-
type: 'options',
|
|
125
|
-
default: 'markdown',
|
|
126
|
-
options: [
|
|
127
|
-
{ name: 'Markdown', value: 'markdown' },
|
|
128
|
-
{ name: 'Text', value: 'text' }
|
|
129
|
-
],
|
|
130
|
-
description: 'Format of the extracted web page content'
|
|
131
|
-
}
|
|
132
35
|
];
|
|
36
|
+
const displayOptions = {
|
|
37
|
+
show: {
|
|
38
|
+
resource: ['crawl'],
|
|
39
|
+
operation: ['crawl'],
|
|
40
|
+
},
|
|
41
|
+
};
|
|
42
|
+
exports.description = (0, display_1.updateDisplayOptions)(displayOptions, exports.properties);
|
|
133
43
|
async function execute(index) {
|
|
134
|
-
const
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
140
|
-
}
|
|
44
|
+
const url = this.getNodeParameter('url', index);
|
|
45
|
+
const options = this.getNodeParameter('options', index);
|
|
46
|
+
const body = {
|
|
47
|
+
url,
|
|
48
|
+
...options,
|
|
49
|
+
};
|
|
141
50
|
const responseData = await transport_1.tavilyApiRequest.call(this, 'POST', '/crawl', body);
|
|
142
51
|
return this.helpers.returnJsonArray([responseData]);
|
|
143
52
|
}
|
|
144
53
|
exports.execute = execute;
|
|
145
|
-
exports.description = exports.properties;
|
|
146
54
|
//# sourceMappingURL=crawl.operation.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"crawl.operation.js","sourceRoot":"","sources":["../../../../../nodes/Tavily/actions/crawl/crawl.operation.ts"],"names":[],"mappings":";;;AACA,+CAAmD;
|
|
1
|
+
{"version":3,"file":"crawl.operation.js","sourceRoot":"","sources":["../../../../../nodes/Tavily/actions/crawl/crawl.operation.ts"],"names":[],"mappings":";;;AACA,+CAAmD;AACnD,gFAAsE;AACtE,2CAAqD;AAExC,QAAA,UAAU,GAAsB;IAC3C;QACE,WAAW,EAAE,KAAK;QAClB,IAAI,EAAE,KAAK;QACX,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,IAAI;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,iCAAiC;QAC9C,WAAW,EAAE,yBAAyB;QACtC,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,QAAQ,EAAE,CAAC,OAAO,CAAC;aACpB;SACF;KACF;IACD;QACE,WAAW,EAAE,SAAS;QACtB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,YAAY;QACzB,OAAO,EAAE,EAAE;QACX,OAAO,EAAE,kCAAY,CAAC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,MAAM,CAAC,IAAI,KAAK,KAAK,CAAC;QAC7D,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,QAAQ,EAAE,CAAC,OAAO,CAAC;aACpB;SACF;KACF;CACF,CAAC;AAEF,MAAM,cAAc,GAAG;IACrB,IAAI,EAAE;QACJ,QAAQ,EAAE,CAAC,OAAO,CAAC;QACnB,SAAS,EAAE,CAAC,OAAO,CAAC;KACrB;CACF,CAAC;AAEW,QAAA,WAAW,GAAG,IAAA,8BAAoB,EAAC,cAAc,EAAE,kBAAU,CAAC,CAAC;AAErE,KAAK,UAAU,OAAO,CAA0B,KAAa;IAClE,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAW,CAAC;IAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAgB,CAAC;IAEvE,MAAM,IAAI,GAAgB;QACxB,GAAG;QACH,GAAG,OAAO;KACX,CAAC;IAEF,MAAM,YAAY,GAAG,MAAM,4BAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC/E,OAAO,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;AACtD,CAAC;AAXD,0BAWC"}
|
|
@@ -23,9 +23,9 @@ var __importStar = (this && this.__importStar) || function (mod) {
|
|
|
23
23
|
return result;
|
|
24
24
|
};
|
|
25
25
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
26
|
-
exports.description = exports.
|
|
27
|
-
const
|
|
28
|
-
exports.
|
|
26
|
+
exports.description = exports.url = void 0;
|
|
27
|
+
const url = __importStar(require("./url.operation"));
|
|
28
|
+
exports.url = url;
|
|
29
29
|
exports.description = [
|
|
30
30
|
{
|
|
31
31
|
displayName: 'Operation',
|
|
@@ -39,14 +39,14 @@ exports.description = [
|
|
|
39
39
|
},
|
|
40
40
|
options: [
|
|
41
41
|
{
|
|
42
|
-
name: '
|
|
43
|
-
value: '
|
|
44
|
-
description: '
|
|
42
|
+
name: 'URL',
|
|
43
|
+
value: 'url',
|
|
44
|
+
description: 'The root URL to begin the crawl',
|
|
45
45
|
action: 'Crawl',
|
|
46
46
|
},
|
|
47
47
|
],
|
|
48
|
-
default: '
|
|
48
|
+
default: 'url',
|
|
49
49
|
},
|
|
50
|
-
...
|
|
50
|
+
...url.description,
|
|
51
51
|
];
|
|
52
52
|
//# sourceMappingURL=index.js.map
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../nodes/Tavily/actions/crawl/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AACA,
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../nodes/Tavily/actions/crawl/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;AACA,qDAAuC;AAE9B,kBAAG;AAEC,QAAA,WAAW,GAAsB;IAC5C;QACE,WAAW,EAAE,WAAW;QACxB,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,SAAS;QACf,gBAAgB,EAAE,IAAI;QACtB,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,QAAQ,EAAE,CAAC,OAAO,CAAC;aACpB;SACF;QACD,OAAO,EAAE;YACP;gBACE,IAAI,EAAE,KAAK;gBACX,KAAK,EAAE,KAAK;gBACZ,WAAW,EAAE,iCAAiC;gBAC9C,MAAM,EAAE,OAAO;aAChB;SACF;QACD,OAAO,EAAE,KAAK;KACf;IACD,GAAG,GAAG,CAAC,WAAW;CACnB,CAAC"}
|
|
@@ -0,0 +1,36 @@
|
|
|
1
|
+
import type { IExecuteFunctions, INodeProperties } from 'n8n-workflow';
|
|
2
|
+
export declare const properties: INodeProperties[];
|
|
3
|
+
export declare const description: {
|
|
4
|
+
displayOptions: {
|
|
5
|
+
hide?: {
|
|
6
|
+
[key: string]: (import("n8n-workflow").NodeParameterValue | import("n8n-workflow").DisplayCondition)[] | undefined;
|
|
7
|
+
} | undefined;
|
|
8
|
+
show?: {
|
|
9
|
+
[key: string]: (import("n8n-workflow").NodeParameterValue | import("n8n-workflow").DisplayCondition)[] | undefined;
|
|
10
|
+
'@version'?: (number | import("n8n-workflow").DisplayCondition)[] | undefined;
|
|
11
|
+
} | undefined;
|
|
12
|
+
hideOnCloud?: boolean | undefined;
|
|
13
|
+
};
|
|
14
|
+
displayName: string;
|
|
15
|
+
name: string;
|
|
16
|
+
type: import("n8n-workflow").NodePropertyTypes;
|
|
17
|
+
typeOptions?: import("n8n-workflow").INodePropertyTypeOptions | undefined;
|
|
18
|
+
default: import("n8n-workflow").NodeParameterValueType;
|
|
19
|
+
description?: string | undefined;
|
|
20
|
+
hint?: string | undefined;
|
|
21
|
+
disabledOptions?: import("n8n-workflow").IDisplayOptions | undefined;
|
|
22
|
+
options?: (INodeProperties | import("n8n-workflow").INodePropertyOptions | import("n8n-workflow").INodePropertyCollection)[] | undefined;
|
|
23
|
+
placeholder?: string | undefined;
|
|
24
|
+
isNodeSetting?: boolean | undefined;
|
|
25
|
+
noDataExpression?: boolean | undefined;
|
|
26
|
+
required?: boolean | undefined;
|
|
27
|
+
routing?: import("n8n-workflow").INodePropertyRouting | undefined;
|
|
28
|
+
credentialTypes?: ("extends:oAuth2Api" | "extends:oAuth1Api" | "has:authenticate" | "has:genericAuth")[] | undefined;
|
|
29
|
+
extractValue?: import("n8n-workflow").INodePropertyValueExtractorRegex | undefined;
|
|
30
|
+
modes?: import("n8n-workflow").INodePropertyMode[] | undefined;
|
|
31
|
+
requiresDataPath?: "single" | "multiple" | undefined;
|
|
32
|
+
doNotInherit?: boolean | undefined;
|
|
33
|
+
validateType?: keyof import("n8n-workflow").FieldTypeMap | undefined;
|
|
34
|
+
ignoreValidationDuringExecution?: boolean | undefined;
|
|
35
|
+
}[];
|
|
36
|
+
export declare function execute(this: IExecuteFunctions, index: number): Promise<import("n8n-workflow").INodeExecutionData[]>;
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.execute = exports.description = exports.properties = void 0;
|
|
4
|
+
const transport_1 = require("../../transport");
|
|
5
|
+
const display_1 = require("../../display");
|
|
6
|
+
const common_descriptions_1 = require("../../descriptions/common.descriptions");
|
|
7
|
+
exports.properties = [
|
|
8
|
+
{
|
|
9
|
+
displayName: 'URL',
|
|
10
|
+
name: 'url',
|
|
11
|
+
description: 'The root URL to begin the crawl',
|
|
12
|
+
type: 'string',
|
|
13
|
+
required: true,
|
|
14
|
+
default: '',
|
|
15
|
+
placeholder: 'https://www.example.com',
|
|
16
|
+
displayOptions: {
|
|
17
|
+
show: {
|
|
18
|
+
resource: ['crawl'],
|
|
19
|
+
},
|
|
20
|
+
},
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
displayName: 'Options',
|
|
24
|
+
name: 'options',
|
|
25
|
+
type: 'collection',
|
|
26
|
+
placeholder: 'Add option',
|
|
27
|
+
default: {},
|
|
28
|
+
options: common_descriptions_1.crawlOptions
|
|
29
|
+
},
|
|
30
|
+
];
|
|
31
|
+
const displayOptions = {
|
|
32
|
+
show: {
|
|
33
|
+
resource: ['crawl'],
|
|
34
|
+
operation: ['url'],
|
|
35
|
+
},
|
|
36
|
+
};
|
|
37
|
+
exports.description = (0, display_1.updateDisplayOptions)(displayOptions, exports.properties);
|
|
38
|
+
async function execute(index) {
|
|
39
|
+
const url = this.getNodeParameter('url', index);
|
|
40
|
+
const options = this.getNodeParameter('options', index);
|
|
41
|
+
const processedOptions = {};
|
|
42
|
+
for (const [key, value] of Object.entries(options)) {
|
|
43
|
+
if (typeof value === 'boolean') {
|
|
44
|
+
processedOptions[key] = value;
|
|
45
|
+
}
|
|
46
|
+
else {
|
|
47
|
+
processedOptions[key] = value;
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
const body = {
|
|
51
|
+
url,
|
|
52
|
+
...processedOptions,
|
|
53
|
+
};
|
|
54
|
+
const responseData = await transport_1.tavilyApiRequest.call(this, 'POST', '/crawl', body);
|
|
55
|
+
return this.helpers.returnJsonArray([responseData]);
|
|
56
|
+
}
|
|
57
|
+
exports.execute = execute;
|
|
58
|
+
//# sourceMappingURL=url.operation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"url.operation.js","sourceRoot":"","sources":["../../../../../nodes/Tavily/actions/crawl/url.operation.ts"],"names":[],"mappings":";;;AACA,+CAAmD;AACnD,2CAAqD;AACrD,gFAAsE;AAEzD,QAAA,UAAU,GAAsB;IAC3C;QACE,WAAW,EAAE,KAAK;QAClB,IAAI,EAAE,KAAK;QACX,WAAW,EAAE,iCAAiC;QAC9C,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,IAAI;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,yBAAyB;QACtC,cAAc,EAAE;YACd,IAAI,EAAE;gBACJ,QAAQ,EAAE,CAAC,OAAO,CAAC;aACpB;SACF;KACF;IACD;QACE,WAAW,EAAE,SAAS;QACtB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,YAAY;QACzB,OAAO,EAAE,EAAE;QACX,OAAO,EAAE,kCAAY;KACvB;CACD,CAAC;AAEF,MAAM,cAAc,GAAG;IACrB,IAAI,EAAE;QACJ,QAAQ,EAAE,CAAC,OAAO,CAAC;QACnB,SAAS,EAAE,CAAC,KAAK,CAAC;KACnB;CACF,CAAC;AAEW,QAAA,WAAW,GAAG,IAAA,8BAAoB,EAAC,cAAc,EAAE,kBAAU,CAAC,CAAC;AAErE,KAAK,UAAU,OAAO,CAA0B,KAAa;IAClE,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,KAAK,CAAW,CAAC;IAC1D,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAgB,CAAC;IAGvE,MAAM,gBAAgB,GAAgB,EAAE,CAAC;IACzC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;QAClD,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE;YAC9B,gBAAgB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;SAC/B;aAAM;YACL,gBAAgB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;SAC/B;KACF;IAED,MAAM,IAAI,GAAgB;QACxB,GAAG;QACH,GAAG,gBAAgB;KACpB,CAAC;IAEF,MAAM,YAAY,GAAG,MAAM,4BAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAC/E,OAAO,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,CAAC,YAAY,CAAC,CAAC,CAAC;AACtD,CAAC;AArBD,0BAqBC"}
|
|
@@ -16,7 +16,12 @@ exports.properties = [
|
|
|
16
16
|
},
|
|
17
17
|
required: true,
|
|
18
18
|
default: [],
|
|
19
|
-
placeholder: 'e.g. https://tavily.com'
|
|
19
|
+
placeholder: 'e.g. https://tavily.com',
|
|
20
|
+
displayOptions: {
|
|
21
|
+
show: {
|
|
22
|
+
resource: ['extract'],
|
|
23
|
+
},
|
|
24
|
+
},
|
|
20
25
|
},
|
|
21
26
|
{
|
|
22
27
|
displayName: 'Options',
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"urls.operation.js","sourceRoot":"","sources":["../../../../../nodes/Tavily/actions/extract/urls.operation.ts"],"names":[],"mappings":";;;AACA,+CAAmD;AACnD,2CAAqD;AACrD,gFAAwE;AAG3D,QAAA,UAAU,GAAsB;IAC5C;QACC,WAAW,EAAE,MAAM;QACnB,IAAI,EAAE,MAAM;QACZ,WAAW,EAAE,wCAAwC;QACrD,IAAI,EAAE,QAAQ;QACd,WAAW,EAAE;YACZ,cAAc,EAAE,IAAI;YACpB,uBAAuB,EAAE,SAAS;SAClC;QACD,QAAQ,EAAE,IAAI;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,yBAAyB;
|
|
1
|
+
{"version":3,"file":"urls.operation.js","sourceRoot":"","sources":["../../../../../nodes/Tavily/actions/extract/urls.operation.ts"],"names":[],"mappings":";;;AACA,+CAAmD;AACnD,2CAAqD;AACrD,gFAAwE;AAG3D,QAAA,UAAU,GAAsB;IAC5C;QACC,WAAW,EAAE,MAAM;QACnB,IAAI,EAAE,MAAM;QACZ,WAAW,EAAE,wCAAwC;QACrD,IAAI,EAAE,QAAQ;QACd,WAAW,EAAE;YACZ,cAAc,EAAE,IAAI;YACpB,uBAAuB,EAAE,SAAS;SAClC;QACD,QAAQ,EAAE,IAAI;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,yBAAyB;QACtC,cAAc,EAAE;YACf,IAAI,EAAE;gBACL,QAAQ,EAAE,CAAC,SAAS,CAAC;aACrB;SACD;KACD;IACD;QACC,WAAW,EAAE,SAAS;QACtB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,YAAY;QACzB,OAAO,EAAE,EAAE;QACX,OAAO,EAAE,oCAAc;KACvB;CACD,CAAC;AAEF,MAAM,cAAc,GAAG;IACtB,IAAI,EAAE;QACL,QAAQ,EAAE,CAAC,SAAS,CAAC;QACrB,SAAS,EAAE,CAAC,MAAM,CAAC;KACnB;CACD,CAAC;AAEW,QAAA,WAAW,GAAG,IAAA,8BAAoB,EAAC,cAAc,EAAE,kBAAU,CAAC,CAAC;AAErE,KAAK,UAAU,OAAO,CAA0B,KAAa;IACnE,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,KAAK,CAAa,CAAC;IAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAgB,CAAC;IAGvE,MAAM,gBAAgB,GAAgB,EAAE,CAAC;IACzC,KAAK,MAAM,CAAC,GAAG,EAAE,KAAK,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE;QACnD,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE;YAC/B,gBAAgB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;SAC9B;aAAM;YACN,gBAAgB,CAAC,GAAG,CAAC,GAAG,KAAK,CAAC;SAC9B;KACD;IAED,MAAM,IAAI,GAAgB;QACzB,MAAM,EAAE,IAAI;QACZ,GAAG,gBAAgB;KACnB,CAAC;IAEF,MAAM,QAAQ,GAAG,UAAU,CAAC;IAE5B,MAAM,YAAY,GAAG,MAAM,4BAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAE/E,OAAO,IAAI,CAAC,OAAO,CAAC,0BAA0B,CAC7C,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,YAA6B,CAAC,EAC3D,EAAC,QAAQ,EAAE,EAAC,IAAI,EAAE,KAAK,EAAC,EAAC,CACzB,CAAC;AACH,CAAC;AA3BD,0BA2BC"}
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"query.operation.js","sourceRoot":"","sources":["../../../../../nodes/Tavily/actions/search/query.operation.ts"],"names":[],"mappings":";;;AACA,+CAAmD;AACnD,2CAAqD;AACrD,qDAAkD;AAGrC,QAAA,UAAU,GAAsB;IAC5C;QACC,WAAW,EAAE,OAAO;QACpB,IAAI,EAAE,OAAO;QACb,WAAW,EAAE,iBAAiB;QAC9B,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,IAAI;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,wBAAwB;
|
|
1
|
+
{"version":3,"file":"query.operation.js","sourceRoot":"","sources":["../../../../../nodes/Tavily/actions/search/query.operation.ts"],"names":[],"mappings":";;;AACA,+CAAmD;AACnD,2CAAqD;AACrD,qDAAkD;AAGrC,QAAA,UAAU,GAAsB;IAC5C;QACC,WAAW,EAAE,OAAO;QACpB,IAAI,EAAE,OAAO;QACb,WAAW,EAAE,iBAAiB;QAC9B,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,IAAI;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,wBAAwB;QACrC,cAAc,EAAE;YACf,IAAI,EAAE;gBACL,QAAQ,EAAE,CAAC,QAAQ,CAAC;aACpB;SACD;KACD;IACD;QACC,WAAW,EAAE,SAAS;QACtB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,YAAY;QAClB,WAAW,EAAE,YAAY;QACzB,OAAO,EAAE,EAAE;QACX,OAAO,EAAE,2BAAY;KACrB;CACD,CAAC;AAEF,MAAM,cAAc,GAAG;IACtB,IAAI,EAAE;QACL,QAAQ,EAAE,CAAC,QAAQ,CAAC;QACpB,SAAS,EAAE,CAAC,OAAO,CAAC;KACpB;CACD,CAAC;AAEW,QAAA,WAAW,GAAG,IAAA,8BAAoB,EAAC,cAAc,EAAE,kBAAU,CAAC,CAAC;AAErE,KAAK,UAAU,OAAO,CAA0B,KAAa;IACnE,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,KAAK,CAAW,CAAC;IAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,KAAK,CAAgB,CAAC;IAEvE,MAAM,IAAI,GAAgB;QACzB,OAAO,EAAE,KAAK;QACd,GAAG,OAAO;KACV,CAAC;IAEF,MAAM,QAAQ,GAAG,SAAS,CAAC;IAE3B,MAAM,YAAY,GAAG,MAAM,4BAAgB,CAAC,IAAI,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,IAAI,CAAC,CAAC;IAE/E,OAAO,IAAI,CAAC,OAAO,CAAC,0BAA0B,CAC7C,IAAI,CAAC,OAAO,CAAC,eAAe,CAAC,YAA6B,CAAC,EAC3D,EAAC,QAAQ,EAAE,EAAC,IAAI,EAAE,KAAK,EAAC,EAAC,CACzB,CAAC;AACH,CAAC;AAjBD,0BAiBC"}
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
"use strict";
|
|
2
2
|
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
-
exports.queryFields = exports.queryOptions = exports.extractOptions = void 0;
|
|
3
|
+
exports.crawlOptions = exports.queryFields = exports.queryOptions = exports.extractOptions = void 0;
|
|
4
4
|
exports.extractOptions = [
|
|
5
5
|
{
|
|
6
6
|
displayName: 'Include Images',
|
|
@@ -26,6 +26,24 @@ exports.extractOptions = [
|
|
|
26
26
|
],
|
|
27
27
|
description: 'The depth of the extraction process. advanced extraction retrieves more data, including tables and embedded content, with higher success but may increase latency.',
|
|
28
28
|
},
|
|
29
|
+
{
|
|
30
|
+
displayName: 'Format',
|
|
31
|
+
name: 'format',
|
|
32
|
+
type: 'options',
|
|
33
|
+
default: 'markdown',
|
|
34
|
+
options: [
|
|
35
|
+
{ name: 'Markdown', value: 'markdown' },
|
|
36
|
+
{ name: 'Text', value: 'text' }
|
|
37
|
+
],
|
|
38
|
+
description: 'The format of the extracted web page content. markdown returns content in markdown format. text returns plain text and may increase latency.'
|
|
39
|
+
},
|
|
40
|
+
{
|
|
41
|
+
displayName: 'Include Favicon',
|
|
42
|
+
name: 'include_favicon',
|
|
43
|
+
type: 'boolean',
|
|
44
|
+
default: false,
|
|
45
|
+
description: 'Whether to include the favicon URL for each result'
|
|
46
|
+
},
|
|
29
47
|
];
|
|
30
48
|
exports.queryOptions = [
|
|
31
49
|
{
|
|
@@ -186,6 +204,13 @@ exports.queryOptions = [
|
|
|
186
204
|
description: 'A list of domains to exclude from the search results',
|
|
187
205
|
placeholder: 'example.com',
|
|
188
206
|
},
|
|
207
|
+
{
|
|
208
|
+
displayName: 'Include Favicon',
|
|
209
|
+
name: 'include_favicon',
|
|
210
|
+
type: 'boolean',
|
|
211
|
+
default: false,
|
|
212
|
+
description: 'Whether to include the favicon URL for each result'
|
|
213
|
+
},
|
|
189
214
|
];
|
|
190
215
|
exports.queryFields = [
|
|
191
216
|
{
|
|
@@ -213,4 +238,130 @@ exports.queryFields = [
|
|
|
213
238
|
value: 'unreadItemCount',
|
|
214
239
|
},
|
|
215
240
|
];
|
|
241
|
+
exports.crawlOptions = [
|
|
242
|
+
{
|
|
243
|
+
displayName: 'Instructions',
|
|
244
|
+
name: 'instructions',
|
|
245
|
+
type: 'string',
|
|
246
|
+
default: '',
|
|
247
|
+
description: 'Natural language instructions for the crawler'
|
|
248
|
+
},
|
|
249
|
+
{
|
|
250
|
+
displayName: 'Max Depth',
|
|
251
|
+
name: 'max_depth',
|
|
252
|
+
type: 'number',
|
|
253
|
+
default: 1,
|
|
254
|
+
description: 'Max depth of the crawl',
|
|
255
|
+
typeOptions: { minValue: 1 }
|
|
256
|
+
},
|
|
257
|
+
{
|
|
258
|
+
displayName: 'Max Breadth',
|
|
259
|
+
name: 'max_breadth',
|
|
260
|
+
type: 'number',
|
|
261
|
+
default: 20,
|
|
262
|
+
description: 'Max number of links to follow per level',
|
|
263
|
+
typeOptions: { minValue: 1 }
|
|
264
|
+
},
|
|
265
|
+
{
|
|
266
|
+
displayName: 'Limit',
|
|
267
|
+
name: 'limit',
|
|
268
|
+
type: 'number',
|
|
269
|
+
default: 50,
|
|
270
|
+
description: 'Max number of results to return',
|
|
271
|
+
typeOptions: { minValue: 1 }
|
|
272
|
+
},
|
|
273
|
+
{
|
|
274
|
+
displayName: 'Categories',
|
|
275
|
+
name: 'categories',
|
|
276
|
+
type: 'multiOptions',
|
|
277
|
+
default: [],
|
|
278
|
+
options: [
|
|
279
|
+
{ name: 'About', value: 'About' },
|
|
280
|
+
{ name: 'Blog', value: 'Blog' },
|
|
281
|
+
{ name: 'Careers', value: 'Careers' },
|
|
282
|
+
{ name: 'Community', value: 'Community' },
|
|
283
|
+
{ name: 'Contact', value: 'Contact' },
|
|
284
|
+
{ name: 'Developers', value: 'Developers' },
|
|
285
|
+
{ name: 'Documentation', value: 'Documentation' },
|
|
286
|
+
{ name: 'Media', value: 'Media' },
|
|
287
|
+
{ name: 'Pricing', value: 'Pricing' },
|
|
288
|
+
],
|
|
289
|
+
description: 'Filter URLs using predefined categories'
|
|
290
|
+
},
|
|
291
|
+
{
|
|
292
|
+
displayName: 'Select Paths',
|
|
293
|
+
name: 'select_paths',
|
|
294
|
+
type: 'string',
|
|
295
|
+
typeOptions: { multipleValues: true },
|
|
296
|
+
default: [],
|
|
297
|
+
description: 'Regex patterns to select only URLs with specific path patterns'
|
|
298
|
+
},
|
|
299
|
+
{
|
|
300
|
+
displayName: 'Select Domains',
|
|
301
|
+
name: 'select_domains',
|
|
302
|
+
type: 'string',
|
|
303
|
+
typeOptions: { multipleValues: true },
|
|
304
|
+
default: [],
|
|
305
|
+
description: 'Regex patterns to select crawling to specific domains or subdomains'
|
|
306
|
+
},
|
|
307
|
+
{
|
|
308
|
+
displayName: 'Exclude Paths',
|
|
309
|
+
name: 'exclude_paths',
|
|
310
|
+
type: 'string',
|
|
311
|
+
typeOptions: { multipleValues: true },
|
|
312
|
+
default: [],
|
|
313
|
+
description: 'Regex patterns to exclude URLs with specific path patterns'
|
|
314
|
+
},
|
|
315
|
+
{
|
|
316
|
+
displayName: 'Exclude Domains',
|
|
317
|
+
name: 'exclude_domains',
|
|
318
|
+
type: 'string',
|
|
319
|
+
typeOptions: { multipleValues: true },
|
|
320
|
+
default: [],
|
|
321
|
+
description: 'Regex patterns to exclude specific domains or subdomains from crawling'
|
|
322
|
+
},
|
|
323
|
+
{
|
|
324
|
+
displayName: 'Allow External',
|
|
325
|
+
name: 'allow_external',
|
|
326
|
+
type: 'boolean',
|
|
327
|
+
default: false,
|
|
328
|
+
description: 'Whether to allow following links that go to external domains'
|
|
329
|
+
},
|
|
330
|
+
{
|
|
331
|
+
displayName: 'Include Images',
|
|
332
|
+
name: 'include_images',
|
|
333
|
+
type: 'boolean',
|
|
334
|
+
default: false,
|
|
335
|
+
description: 'Whether to include images in the crawl results'
|
|
336
|
+
},
|
|
337
|
+
{
|
|
338
|
+
displayName: 'Extract Depth',
|
|
339
|
+
name: 'extract_depth',
|
|
340
|
+
type: 'options',
|
|
341
|
+
default: 'basic',
|
|
342
|
+
options: [
|
|
343
|
+
{ name: 'Basic', value: 'basic' },
|
|
344
|
+
{ name: 'Advanced', value: 'advanced' }
|
|
345
|
+
],
|
|
346
|
+
description: 'Extraction depth'
|
|
347
|
+
},
|
|
348
|
+
{
|
|
349
|
+
displayName: 'Format',
|
|
350
|
+
name: 'format',
|
|
351
|
+
type: 'options',
|
|
352
|
+
default: 'markdown',
|
|
353
|
+
options: [
|
|
354
|
+
{ name: 'Markdown', value: 'markdown' },
|
|
355
|
+
{ name: 'Text', value: 'text' }
|
|
356
|
+
],
|
|
357
|
+
description: 'Format of the extracted web page content'
|
|
358
|
+
},
|
|
359
|
+
{
|
|
360
|
+
displayName: 'Include Favicon',
|
|
361
|
+
name: 'include_favicon',
|
|
362
|
+
type: 'boolean',
|
|
363
|
+
default: false,
|
|
364
|
+
description: 'Whether to include the favicon URL for each result'
|
|
365
|
+
},
|
|
366
|
+
];
|
|
216
367
|
//# sourceMappingURL=common.descriptions.js.map
|