n8n-nodes-google-map-scraper 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +19 -0
- package/README.md +247 -0
- package/dist/icons/googlemeet.svg +4 -0
- package/dist/nodes/GoogleMapsScraper/GoogleMapsScraper.node.d.ts +5 -0
- package/dist/nodes/GoogleMapsScraper/GoogleMapsScraper.node.js +179 -0
- package/dist/nodes/GoogleMapsScraper/GoogleMapsScraper.node.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/generateGrid.operation.d.ts +18 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/generateGrid.operation.js +84 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/generateGrid.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/index.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/index.js +59 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/index.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/lookupBbox.operation.d.ts +11 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/lookupBbox.operation.js +91 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/lookupBbox.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/shrinkBbox.operation.d.ts +11 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/shrinkBbox.operation.js +82 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/shrinkBbox.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/index.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/index.js +54 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/index.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/multiViewportGrid.operation.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/multiViewportGrid.operation.js +78 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/multiViewportGrid.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/singleViewport.operation.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/singleViewport.operation.js +89 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/singleViewport.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/extractPrefetchUrl.operation.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/extractPrefetchUrl.operation.js +89 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/extractPrefetchUrl.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/fetchHtmlShell.operation.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/fetchHtmlShell.operation.js +78 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/fetchHtmlShell.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/index.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/index.js +64 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/index.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/paginateResults.operation.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/paginateResults.operation.js +246 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/paginateResults.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/parseRecords.operation.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/parseRecords.operation.js +72 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/parseRecords.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/googleMapsScraper.svg +5 -0
- package/dist/nodes/GoogleMapsScraper/helpers/pagination.d.ts +5 -0
- package/dist/nodes/GoogleMapsScraper/helpers/pagination.js +29 -0
- package/dist/nodes/GoogleMapsScraper/helpers/pagination.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/helpers/parser.d.ts +18 -0
- package/dist/nodes/GoogleMapsScraper/helpers/parser.js +78 -0
- package/dist/nodes/GoogleMapsScraper/helpers/parser.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/helpers/proxy.d.ts +6 -0
- package/dist/nodes/GoogleMapsScraper/helpers/proxy.js +20 -0
- package/dist/nodes/GoogleMapsScraper/helpers/proxy.js.map +1 -0
- package/dist/package.json +54 -0
- package/dist/tsconfig.tsbuildinfo +1 -0
- package/package.json +54 -0
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.parameters = void 0;
|
|
7
|
+
exports.execute = execute;
|
|
8
|
+
const n8n_workflow_1 = require("n8n-workflow");
|
|
9
|
+
const axios_1 = __importDefault(require("axios"));
|
|
10
|
+
exports.parameters = [
|
|
11
|
+
{
|
|
12
|
+
displayName: 'City Name',
|
|
13
|
+
name: 'cityName',
|
|
14
|
+
type: 'string',
|
|
15
|
+
default: '',
|
|
16
|
+
required: true,
|
|
17
|
+
description: 'City name to lookup (e.g., "Bangalore", "New York")',
|
|
18
|
+
displayOptions: {
|
|
19
|
+
show: { resource: ['city'], operation: ['lookupBbox'] },
|
|
20
|
+
},
|
|
21
|
+
},
|
|
22
|
+
];
|
|
23
|
+
async function execute() {
|
|
24
|
+
const items = this.getInputData();
|
|
25
|
+
const results = [];
|
|
26
|
+
try {
|
|
27
|
+
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
|
28
|
+
const cityName = this.getNodeParameter('cityName', itemIndex);
|
|
29
|
+
try {
|
|
30
|
+
const response = await axios_1.default.get('https://nominatim.openstreetmap.org/search', {
|
|
31
|
+
params: {
|
|
32
|
+
q: cityName,
|
|
33
|
+
format: 'json',
|
|
34
|
+
limit: 1,
|
|
35
|
+
},
|
|
36
|
+
headers: {
|
|
37
|
+
'user-agent': 'gmaps-scraper-node/1.0',
|
|
38
|
+
},
|
|
39
|
+
timeout: 10000,
|
|
40
|
+
});
|
|
41
|
+
if (!response.data || response.data.length === 0) {
|
|
42
|
+
results.push({
|
|
43
|
+
json: {
|
|
44
|
+
ok: false,
|
|
45
|
+
error: `City not found: ${cityName}`,
|
|
46
|
+
city: cityName,
|
|
47
|
+
},
|
|
48
|
+
});
|
|
49
|
+
continue;
|
|
50
|
+
}
|
|
51
|
+
const result = response.data[0];
|
|
52
|
+
const bbox = {
|
|
53
|
+
north: parseFloat(result.boundingbox[1]),
|
|
54
|
+
south: parseFloat(result.boundingbox[0]),
|
|
55
|
+
east: parseFloat(result.boundingbox[3]),
|
|
56
|
+
west: parseFloat(result.boundingbox[2]),
|
|
57
|
+
lat: parseFloat(result.lat),
|
|
58
|
+
lng: parseFloat(result.lon),
|
|
59
|
+
};
|
|
60
|
+
results.push({
|
|
61
|
+
json: {
|
|
62
|
+
ok: true,
|
|
63
|
+
bbox,
|
|
64
|
+
city: result.name || cityName,
|
|
65
|
+
display_name: result.display_name,
|
|
66
|
+
location: {
|
|
67
|
+
lat: bbox.lat,
|
|
68
|
+
lng: bbox.lng,
|
|
69
|
+
},
|
|
70
|
+
},
|
|
71
|
+
});
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
75
|
+
results.push({
|
|
76
|
+
json: {
|
|
77
|
+
ok: false,
|
|
78
|
+
error: msg,
|
|
79
|
+
city: cityName,
|
|
80
|
+
},
|
|
81
|
+
});
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return [results];
|
|
85
|
+
}
|
|
86
|
+
catch (error) {
|
|
87
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
88
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Lookup BBox operation failed: ${msg}`);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
//# sourceMappingURL=lookupBbox.operation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"lookupBbox.operation.js","sourceRoot":"","sources":["../../../../../nodes/GoogleMapsScraper/actions/city/lookupBbox.operation.ts"],"names":[],"mappings":";;;;;;AAiCA,0BAwEC;AAzGD,+CAA0G;AAG1G,kDAA0B;AAWb,QAAA,UAAU,GAAsB;IAC3C;QACE,WAAW,EAAE,WAAW;QACxB,IAAI,EAAE,UAAU;QAChB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,EAAE;QACX,QAAQ,EAAE,IAAI;QACd,WAAW,EAAE,qDAAqD;QAClE,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,YAAY,CAAC,EAAE;SACxD;KACF;CACF,CAAC;AAOK,KAAK,UAAU,OAAO;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;IAClC,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,IAAI,CAAC;QACH,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YAC9D,MAAM,QAAQ,GAAG,IAAI,CAAC,gBAAgB,CAAC,UAAU,EAAE,SAAS,CAAW,CAAC;YAExE,IAAI,CAAC;gBAEH,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,GAAG,CAAC,4CAA4C,EAAE;oBAC7E,MAAM,EAAE;wBACN,CAAC,EAAE,QAAQ;wBACX,MAAM,EAAE,MAAM;wBACd,KAAK,EAAE,CAAC;qBACT;oBACD,OAAO,EAAE;wBACP,YAAY,EAAE,wBAAwB;qBACvC;oBACD,OAAO,EAAE,KAAK;iBACf,CAAC,CAAC;gBAEH,IAAI,CAAC,QAAQ,CAAC,IAAI,IAAI,QAAQ,CAAC,IAAI,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;oBACjD,OAAO,CAAC,IAAI,CAAC;wBACX,IAAI,EAAE;4BACJ,EAAE,EAAE,KAAK;4BACT,KAAK,EAAE,mBAAmB,QAAQ,EAAE;4BACpC,IAAI,EAAE,QAAQ;yBACf;qBACF,CAAC,CAAC;oBACH,SAAS;gBACX,CAAC;gBAED,MAAM,MAAM,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;gBAChC,MAAM,IAAI,GAAgB;oBACxB,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;oBACxC,KAAK,EAAE,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;oBACxC,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;oBACvC,IAAI,EAAE,UAAU,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;oBACvC,GAAG,EAAE,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC;oBAC3B,GAAG,EAAE,UAAU,CAAC,MAAM,CAAC,GAAG,CAAC;iBAC5B,CAAC;gBAEF,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE;wBACJ,EAAE,EAAE,IAAI;wBACR,IAAI;wBACJ,IAAI,EAAE,MAAM,CAAC,IAAI,IAAI,QAAQ;wBAC7B,YAAY,EAAE,MAAM,CAAC,YAAY;wBACjC,QAAQ,EAAE;4BACR,GAAG,EAAE,IAAI,CAAC,GAAG;4BACb,GAAG,EAAE,IAAI,CAAC,GAAG;yBACd;qBACF;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnE,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE;wBACJ,EAAE,EAAE,KAAK;wBACT,KAAK,EAAE,GAAG;wBACV,IAAI,EAAE,QAAQ;qBACf;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnE,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,iCAAiC,GAAG,EAAE,CAAC,CAAC;IACvF,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import { IExecuteFunctions, INodeProperties, INodeExecutionData } from 'n8n-workflow';
|
|
2
|
+
export interface BoundingBox {
|
|
3
|
+
north: number;
|
|
4
|
+
south: number;
|
|
5
|
+
east: number;
|
|
6
|
+
west: number;
|
|
7
|
+
lat?: number;
|
|
8
|
+
lng?: number;
|
|
9
|
+
}
|
|
10
|
+
export declare const parameters: INodeProperties[];
|
|
11
|
+
export declare function execute(this: IExecuteFunctions): Promise<INodeExecutionData[][]>;
|
|
@@ -0,0 +1,82 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parameters = void 0;
|
|
4
|
+
exports.execute = execute;
|
|
5
|
+
const n8n_workflow_1 = require("n8n-workflow");
|
|
6
|
+
exports.parameters = [
|
|
7
|
+
{
|
|
8
|
+
displayName: 'Shrink Percentage',
|
|
9
|
+
name: 'shrinkPercentage',
|
|
10
|
+
type: 'number',
|
|
11
|
+
default: 10,
|
|
12
|
+
description: 'Percentage to shrink from edges (0-50)',
|
|
13
|
+
displayOptions: {
|
|
14
|
+
show: { resource: ['city'], operation: ['shrinkBbox'] },
|
|
15
|
+
},
|
|
16
|
+
},
|
|
17
|
+
];
|
|
18
|
+
async function execute() {
|
|
19
|
+
const items = this.getInputData();
|
|
20
|
+
const results = [];
|
|
21
|
+
try {
|
|
22
|
+
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
|
23
|
+
const shrinkPercentage = this.getNodeParameter('shrinkPercentage', itemIndex);
|
|
24
|
+
const item = items[itemIndex].json;
|
|
25
|
+
const bbox = item.bbox;
|
|
26
|
+
if (!bbox || typeof bbox.north !== 'number') {
|
|
27
|
+
results.push({
|
|
28
|
+
json: {
|
|
29
|
+
ok: false,
|
|
30
|
+
error: 'Invalid bounding box in input',
|
|
31
|
+
shrink_percentage: shrinkPercentage,
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
continue;
|
|
35
|
+
}
|
|
36
|
+
if (shrinkPercentage < 0 || shrinkPercentage > 50) {
|
|
37
|
+
results.push({
|
|
38
|
+
json: {
|
|
39
|
+
ok: false,
|
|
40
|
+
error: 'Shrink percentage must be between 0 and 50',
|
|
41
|
+
shrink_percentage: shrinkPercentage,
|
|
42
|
+
},
|
|
43
|
+
});
|
|
44
|
+
continue;
|
|
45
|
+
}
|
|
46
|
+
const shrunkenBbox = shrinkBoundingBox(bbox, shrinkPercentage);
|
|
47
|
+
results.push({
|
|
48
|
+
json: {
|
|
49
|
+
ok: true,
|
|
50
|
+
bbox: shrunkenBbox,
|
|
51
|
+
original_bbox: bbox,
|
|
52
|
+
shrink_percentage: shrinkPercentage,
|
|
53
|
+
removed_area_pct: (shrinkPercentage * 2).toFixed(1),
|
|
54
|
+
},
|
|
55
|
+
});
|
|
56
|
+
}
|
|
57
|
+
return [results];
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
61
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Shrink BBox operation failed: ${msg}`);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
function shrinkBoundingBox(bbox, percentage) {
|
|
65
|
+
const factor = percentage / 100;
|
|
66
|
+
const latRange = bbox.north - bbox.south;
|
|
67
|
+
const lngRange = bbox.east - bbox.west;
|
|
68
|
+
const latReduction = latRange * factor;
|
|
69
|
+
const lngReduction = lngRange * factor;
|
|
70
|
+
const shrunken = {
|
|
71
|
+
south: bbox.south + latReduction,
|
|
72
|
+
north: bbox.north - latReduction,
|
|
73
|
+
west: bbox.west + lngReduction,
|
|
74
|
+
east: bbox.east - lngReduction,
|
|
75
|
+
};
|
|
76
|
+
if (bbox.lat !== undefined && bbox.lng !== undefined) {
|
|
77
|
+
shrunken.lat = (shrunken.south + shrunken.north) / 2;
|
|
78
|
+
shrunken.lng = (shrunken.west + shrunken.east) / 2;
|
|
79
|
+
}
|
|
80
|
+
return shrunken;
|
|
81
|
+
}
|
|
82
|
+
//# sourceMappingURL=shrinkBbox.operation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"shrinkBbox.operation.js","sourceRoot":"","sources":["../../../../../nodes/GoogleMapsScraper/actions/city/shrinkBbox.operation.ts"],"names":[],"mappings":";;;AA6BA,0BAoDC;AAjFD,+CAAuH;AAW1G,QAAA,UAAU,GAAsB;IAC3C;QACE,WAAW,EAAE,mBAAmB;QAChC,IAAI,EAAE,kBAAkB;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,wCAAwC;QACrD,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,MAAM,CAAC,EAAE,SAAS,EAAE,CAAC,YAAY,CAAC,EAAE;SACxD;KACF;CACF,CAAC;AAOK,KAAK,UAAU,OAAO;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;IAClC,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,IAAI,CAAC;QACH,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YAC9D,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,SAAS,CAAW,CAAC;YACxF,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,IAAmB,CAAC;YAElD,MAAM,IAAI,GAAG,IAAI,CAAC,IAAmB,CAAC;YAEtC,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,CAAC,KAAK,KAAK,QAAQ,EAAE,CAAC;gBAC5C,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE;wBACJ,EAAE,EAAE,KAAK;wBACT,KAAK,EAAE,+BAA+B;wBACtC,iBAAiB,EAAE,gBAAgB;qBACpC;iBACF,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAGD,IAAI,gBAAgB,GAAG,CAAC,IAAI,gBAAgB,GAAG,EAAE,EAAE,CAAC;gBAClD,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE;wBACJ,EAAE,EAAE,KAAK;wBACT,KAAK,EAAE,4CAA4C;wBACnD,iBAAiB,EAAE,gBAAgB;qBACpC;iBACF,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,MAAM,YAAY,GAAG,iBAAiB,CAAC,IAAI,EAAE,gBAAgB,CAAC,CAAC;YAE/D,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE;oBACJ,EAAE,EAAE,IAAI;oBACR,IAAI,EAAE,YAAY;oBAClB,aAAa,EAAE,IAAI;oBACnB,iBAAiB,EAAE,gBAAgB;oBACnC,gBAAgB,EAAE,CAAC,gBAAgB,GAAG,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC;iBACpD;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnE,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,iCAAiC,GAAG,EAAE,CAAC,CAAC;IACvF,CAAC;AACH,CAAC;AAQD,SAAS,iBAAiB,CAAC,IAAiB,EAAE,UAAkB;IAC9D,MAAM,MAAM,GAAG,UAAU,GAAG,GAAG,CAAC;IAEhC,MAAM,QAAQ,GAAG,IAAI,CAAC,KAAK,GAAG,IAAI,CAAC,KAAK,CAAC;IACzC,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,GAAG,IAAI,CAAC,IAAI,CAAC;IAEvC,MAAM,YAAY,GAAG,QAAQ,GAAG,MAAM,CAAC;IACvC,MAAM,YAAY,GAAG,QAAQ,GAAG,MAAM,CAAC;IAEvC,MAAM,QAAQ,GAAgB;QAC5B,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,YAAY;QAChC,KAAK,EAAE,IAAI,CAAC,KAAK,GAAG,YAAY;QAChC,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,YAAY;QAC9B,IAAI,EAAE,IAAI,CAAC,IAAI,GAAG,YAAY;KAC/B,CAAC;IAGF,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,IAAI,IAAI,CAAC,GAAG,KAAK,SAAS,EAAE,CAAC;QACrD,QAAQ,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,KAAK,GAAG,QAAQ,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC;QACrD,QAAQ,CAAC,GAAG,GAAG,CAAC,QAAQ,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,QAAQ,CAAC;AAClB,CAAC"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.parameters = void 0;
|
|
37
|
+
exports.execute = execute;
|
|
38
|
+
const n8n_workflow_1 = require("n8n-workflow");
|
|
39
|
+
const singleViewport = __importStar(require("./singleViewport.operation"));
|
|
40
|
+
const multiViewportGrid = __importStar(require("./multiViewportGrid.operation"));
|
|
41
|
+
exports.parameters = [
|
|
42
|
+
...singleViewport.parameters,
|
|
43
|
+
...multiViewportGrid.parameters,
|
|
44
|
+
];
|
|
45
|
+
async function execute(operation) {
|
|
46
|
+
if (operation === 'singleViewport') {
|
|
47
|
+
return singleViewport.execute.call(this);
|
|
48
|
+
}
|
|
49
|
+
if (operation === 'multiViewportGrid') {
|
|
50
|
+
return multiViewportGrid.execute.call(this);
|
|
51
|
+
}
|
|
52
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Unknown scrape operation: ${operation}`);
|
|
53
|
+
}
|
|
54
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../nodes/GoogleMapsScraper/actions/scrape/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AASA,0BAWC;AApBD,+CAA0G;AAC1G,2EAA6D;AAC7D,iFAAmE;AAEtD,QAAA,UAAU,GAAsB;IAC3C,GAAG,cAAc,CAAC,UAAU;IAC5B,GAAG,iBAAiB,CAAC,UAAU;CAChC,CAAC;AAEK,KAAK,UAAU,OAAO,CAE3B,SAAiB;IAEjB,IAAI,SAAS,KAAK,gBAAgB,EAAE,CAAC;QACnC,OAAO,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;IACD,IAAI,SAAS,KAAK,mBAAmB,EAAE,CAAC;QACtC,OAAO,iBAAiB,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC9C,CAAC;IACD,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,6BAA6B,SAAS,EAAE,CAAC,CAAC;AACzF,CAAC"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parameters = void 0;
|
|
4
|
+
exports.execute = execute;
|
|
5
|
+
const n8n_workflow_1 = require("n8n-workflow");
|
|
6
|
+
exports.parameters = [
|
|
7
|
+
{
|
|
8
|
+
displayName: 'Query',
|
|
9
|
+
name: 'query',
|
|
10
|
+
type: 'string',
|
|
11
|
+
default: '',
|
|
12
|
+
required: true,
|
|
13
|
+
description: 'Search query (e.g., "restaurants")',
|
|
14
|
+
displayOptions: {
|
|
15
|
+
show: { resource: ['scrape'], operation: ['multiViewportGrid'] },
|
|
16
|
+
},
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
displayName: 'Parallel Requests',
|
|
20
|
+
name: 'parallelRequests',
|
|
21
|
+
type: 'number',
|
|
22
|
+
default: 3,
|
|
23
|
+
description: 'Number of concurrent viewport requests',
|
|
24
|
+
displayOptions: {
|
|
25
|
+
show: { resource: ['scrape'], operation: ['multiViewportGrid'] },
|
|
26
|
+
},
|
|
27
|
+
},
|
|
28
|
+
];
|
|
29
|
+
async function execute() {
|
|
30
|
+
const items = this.getInputData();
|
|
31
|
+
const results = [];
|
|
32
|
+
try {
|
|
33
|
+
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
|
34
|
+
const query = this.getNodeParameter('query', itemIndex);
|
|
35
|
+
const parallelRequests = this.getNodeParameter('parallelRequests', itemIndex);
|
|
36
|
+
const item = items[itemIndex].json;
|
|
37
|
+
const viewports = item.viewports || [];
|
|
38
|
+
if (!Array.isArray(viewports) || viewports.length === 0) {
|
|
39
|
+
results.push({
|
|
40
|
+
json: {
|
|
41
|
+
ok: false,
|
|
42
|
+
error: 'No viewports provided',
|
|
43
|
+
query,
|
|
44
|
+
},
|
|
45
|
+
});
|
|
46
|
+
continue;
|
|
47
|
+
}
|
|
48
|
+
const scrapeTasks = viewports.map((viewport, index) => {
|
|
49
|
+
const vp = viewport;
|
|
50
|
+
return {
|
|
51
|
+
viewport_index: index,
|
|
52
|
+
cell_id: vp['cell_id'] || `cell_${index}`,
|
|
53
|
+
lat: vp['lat'] || 0,
|
|
54
|
+
lng: vp['lng'] || 0,
|
|
55
|
+
bbox: vp,
|
|
56
|
+
query,
|
|
57
|
+
};
|
|
58
|
+
});
|
|
59
|
+
results.push({
|
|
60
|
+
json: {
|
|
61
|
+
step: 'multi_viewport_grid_start',
|
|
62
|
+
query,
|
|
63
|
+
total_viewports: viewports.length,
|
|
64
|
+
parallel_requests: parallelRequests,
|
|
65
|
+
scrape_tasks: scrapeTasks,
|
|
66
|
+
instructions: 'Run each task through: fetchHtmlShell → extractPrefetchUrl → paginateResults',
|
|
67
|
+
execution_strategy: 'parallel',
|
|
68
|
+
},
|
|
69
|
+
});
|
|
70
|
+
}
|
|
71
|
+
return [results];
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
75
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Multi-viewport grid operation failed: ${msg}`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=multiViewportGrid.operation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"multiViewportGrid.operation.js","sourceRoot":"","sources":["../../../../../nodes/GoogleMapsScraper/actions/scrape/multiViewportGrid.operation.ts"],"names":[],"mappings":";;;AAkCA,0BAsDC;AAxFD,+CAAuH;AAE1G,QAAA,UAAU,GAAsB;IAC3C;QACE,WAAW,EAAE,OAAO;QACpB,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,EAAE;QACX,QAAQ,EAAE,IAAI;QACd,WAAW,EAAE,oCAAoC;QACjD,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC,mBAAmB,CAAC,EAAE;SACjE;KACF;IACD;QACE,WAAW,EAAE,mBAAmB;QAChC,IAAI,EAAE,kBAAkB;QACxB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,CAAC;QACV,WAAW,EAAE,wCAAwC;QACrD,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC,mBAAmB,CAAC,EAAE;SACjE;KACF;CACF,CAAC;AAUK,KAAK,UAAU,OAAO;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;IAClC,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,IAAI,CAAC;QACH,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,SAAS,CAAW,CAAC;YAClE,MAAM,gBAAgB,GAAG,IAAI,CAAC,gBAAgB,CAAC,kBAAkB,EAAE,SAAS,CAAW,CAAC;YACxF,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,IAAmB,CAAC;YAElD,MAAM,SAAS,GAAI,IAAI,CAAC,SAA4C,IAAI,EAAE,CAAC;YAE3E,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,SAAS,CAAC,IAAI,SAAS,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxD,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE;wBACJ,EAAE,EAAE,KAAK;wBACT,KAAK,EAAE,uBAAuB;wBAC9B,KAAK;qBACN;iBACF,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAGD,MAAM,WAAW,GAAG,SAAS,CAAC,GAAG,CAAC,CAAC,QAAQ,EAAE,KAAa,EAAE,EAAE;gBAC5D,MAAM,EAAE,GAAG,QAAmC,CAAC;gBAC/C,OAAO;oBACL,cAAc,EAAE,KAAK;oBACrB,OAAO,EAAG,EAAE,CAAC,SAAS,CAAY,IAAI,QAAQ,KAAK,EAAE;oBACrD,GAAG,EAAG,EAAE,CAAC,KAAK,CAAY,IAAI,CAAC;oBAC/B,GAAG,EAAG,EAAE,CAAC,KAAK,CAAY,IAAI,CAAC;oBAC/B,IAAI,EAAE,EAAE;oBACR,KAAK;iBACN,CAAC;YACJ,CAAC,CAAC,CAAC;YAEH,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE;oBACJ,IAAI,EAAE,2BAA2B;oBACjC,KAAK;oBACL,eAAe,EAAE,SAAS,CAAC,MAAM;oBACjC,iBAAiB,EAAE,gBAAgB;oBACnC,YAAY,EAAE,WAAW;oBACzB,YAAY,EAAE,8EAA8E;oBAC5F,kBAAkB,EAAE,UAAU;iBAC/B;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnE,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,yCAAyC,GAAG,EAAE,CAAC,CAAC;IAC/F,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parameters = void 0;
|
|
4
|
+
exports.execute = execute;
|
|
5
|
+
const n8n_workflow_1 = require("n8n-workflow");
|
|
6
|
+
exports.parameters = [
|
|
7
|
+
{
|
|
8
|
+
displayName: 'Query',
|
|
9
|
+
name: 'query',
|
|
10
|
+
type: 'string',
|
|
11
|
+
default: '',
|
|
12
|
+
required: true,
|
|
13
|
+
description: 'Search query (e.g., "restaurants", "hotels")',
|
|
14
|
+
displayOptions: {
|
|
15
|
+
show: { resource: ['scrape'], operation: ['singleViewport'] },
|
|
16
|
+
},
|
|
17
|
+
},
|
|
18
|
+
{
|
|
19
|
+
displayName: 'Latitude',
|
|
20
|
+
name: 'lat',
|
|
21
|
+
type: 'number',
|
|
22
|
+
default: 0,
|
|
23
|
+
displayOptions: {
|
|
24
|
+
show: { resource: ['scrape'], operation: ['singleViewport'] },
|
|
25
|
+
},
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
displayName: 'Longitude',
|
|
29
|
+
name: 'lng',
|
|
30
|
+
type: 'number',
|
|
31
|
+
default: 0,
|
|
32
|
+
displayOptions: {
|
|
33
|
+
show: { resource: ['scrape'], operation: ['singleViewport'] },
|
|
34
|
+
},
|
|
35
|
+
},
|
|
36
|
+
{
|
|
37
|
+
displayName: 'Bounding Box',
|
|
38
|
+
name: 'bbox',
|
|
39
|
+
type: 'json',
|
|
40
|
+
default: {},
|
|
41
|
+
description: 'Bounding box for viewport',
|
|
42
|
+
displayOptions: {
|
|
43
|
+
show: { resource: ['scrape'], operation: ['singleViewport'] },
|
|
44
|
+
},
|
|
45
|
+
},
|
|
46
|
+
{
|
|
47
|
+
displayName: 'Max Results',
|
|
48
|
+
name: 'maxResults',
|
|
49
|
+
type: 'number',
|
|
50
|
+
default: 240,
|
|
51
|
+
displayOptions: {
|
|
52
|
+
show: { resource: ['scrape'], operation: ['singleViewport'] },
|
|
53
|
+
},
|
|
54
|
+
},
|
|
55
|
+
];
|
|
56
|
+
async function execute() {
|
|
57
|
+
const items = this.getInputData();
|
|
58
|
+
const results = [];
|
|
59
|
+
try {
|
|
60
|
+
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
|
61
|
+
const query = this.getNodeParameter('query', itemIndex);
|
|
62
|
+
const lat = this.getNodeParameter('lat', itemIndex);
|
|
63
|
+
const lng = this.getNodeParameter('lng', itemIndex);
|
|
64
|
+
const maxResults = this.getNodeParameter('maxResults', itemIndex);
|
|
65
|
+
const searchUrl = buildSearchUrl(query, lat, lng);
|
|
66
|
+
results.push({
|
|
67
|
+
json: {
|
|
68
|
+
step: 'single_viewport_start',
|
|
69
|
+
query,
|
|
70
|
+
lat,
|
|
71
|
+
lng,
|
|
72
|
+
search_url: searchUrl,
|
|
73
|
+
max_results: maxResults,
|
|
74
|
+
instructions: 'Pass this through fetchHtmlShell → extractPrefetchUrl → paginateResults',
|
|
75
|
+
},
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
return [results];
|
|
79
|
+
}
|
|
80
|
+
catch (error) {
|
|
81
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
82
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Single viewport operation failed: ${msg}`);
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
function buildSearchUrl(query, lat, lng) {
|
|
86
|
+
const encodedQuery = encodeURIComponent(query);
|
|
87
|
+
return `https://www.google.com/maps/search/${encodedQuery}/@${lat},${lng},15z`;
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=singleViewport.operation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"singleViewport.operation.js","sourceRoot":"","sources":["../../../../../nodes/GoogleMapsScraper/actions/scrape/singleViewport.operation.ts"],"names":[],"mappings":";;;AA6DA,0BAgCC;AA7FD,+CAA0G;AAE7F,QAAA,UAAU,GAAsB;IAC3C;QACE,WAAW,EAAE,OAAO;QACpB,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,EAAE;QACX,QAAQ,EAAE,IAAI;QACd,WAAW,EAAE,8CAA8C;QAC3D,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC,gBAAgB,CAAC,EAAE;SAC9D;KACF;IACD;QACE,WAAW,EAAE,UAAU;QACvB,IAAI,EAAE,KAAK;QACX,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,CAAC;QACV,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC,gBAAgB,CAAC,EAAE;SAC9D;KACF;IACD;QACE,WAAW,EAAE,WAAW;QACxB,IAAI,EAAE,KAAK;QACX,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,CAAC;QACV,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC,gBAAgB,CAAC,EAAE;SAC9D;KACF;IACD;QACE,WAAW,EAAE,cAAc;QAC3B,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,MAAM;QACZ,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,2BAA2B;QACxC,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC,gBAAgB,CAAC,EAAE;SAC9D;KACF;IACD;QACE,WAAW,EAAE,aAAa;QAC1B,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,GAAG;QACZ,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,QAAQ,CAAC,EAAE,SAAS,EAAE,CAAC,gBAAgB,CAAC,EAAE;SAC9D;KACF;CACF,CAAC;AAUK,KAAK,UAAU,OAAO;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;IAClC,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,IAAI,CAAC;QACH,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YAC9D,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,SAAS,CAAW,CAAC;YAClE,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAW,CAAC;YAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAW,CAAC;YAC9D,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,SAAS,CAAW,CAAC;YAG5E,MAAM,SAAS,GAAG,cAAc,CAAC,KAAK,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC;YAElD,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE;oBACJ,IAAI,EAAE,uBAAuB;oBAC7B,KAAK;oBACL,GAAG;oBACH,GAAG;oBACH,UAAU,EAAE,SAAS;oBACrB,WAAW,EAAE,UAAU;oBACvB,YAAY,EAAE,yEAAyE;iBACxF;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnE,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,qCAAqC,GAAG,EAAE,CAAC,CAAC;IAC3F,CAAC;AACH,CAAC;AASD,SAAS,cAAc,CAAC,KAAa,EAAE,GAAW,EAAE,GAAW;IAC7D,MAAM,YAAY,GAAG,kBAAkB,CAAC,KAAK,CAAC,CAAC;IAE/C,OAAO,sCAAsC,YAAY,KAAK,GAAG,IAAI,GAAG,MAAM,CAAC;AACjF,CAAC"}
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parameters = void 0;
|
|
4
|
+
exports.execute = execute;
|
|
5
|
+
const n8n_workflow_1 = require("n8n-workflow");
|
|
6
|
+
exports.parameters = [
|
|
7
|
+
{
|
|
8
|
+
displayName: 'HTML Field',
|
|
9
|
+
name: 'htmlField',
|
|
10
|
+
type: 'string',
|
|
11
|
+
default: 'html',
|
|
12
|
+
description: 'Field containing HTML content',
|
|
13
|
+
displayOptions: {
|
|
14
|
+
show: { resource: ['viewport'], operation: ['extractPrefetchUrl'] },
|
|
15
|
+
},
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
displayName: 'Query',
|
|
19
|
+
name: 'query',
|
|
20
|
+
type: 'string',
|
|
21
|
+
default: '',
|
|
22
|
+
description: 'Search query (e.g., restaurants)',
|
|
23
|
+
displayOptions: {
|
|
24
|
+
show: { resource: ['viewport'], operation: ['extractPrefetchUrl'] },
|
|
25
|
+
},
|
|
26
|
+
},
|
|
27
|
+
];
|
|
28
|
+
function extractPrefetchUrlFromHtml(html) {
|
|
29
|
+
const patterns = [
|
|
30
|
+
/href="([^"]*\/maps\/search[^"]*)"/g,
|
|
31
|
+
/<link[^>]*href="([^"]*\/maps\/search[^"]*)"/g,
|
|
32
|
+
/data-url="([^"]*\/maps\/search[^"]*)"/g,
|
|
33
|
+
];
|
|
34
|
+
for (const pattern of patterns) {
|
|
35
|
+
const match = pattern.exec(html);
|
|
36
|
+
if (match && match[1]) {
|
|
37
|
+
return decodeURIComponent(match[1]);
|
|
38
|
+
}
|
|
39
|
+
}
|
|
40
|
+
return null;
|
|
41
|
+
}
|
|
42
|
+
async function execute() {
|
|
43
|
+
const items = this.getInputData();
|
|
44
|
+
const results = [];
|
|
45
|
+
try {
|
|
46
|
+
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
|
47
|
+
const htmlField = this.getNodeParameter('htmlField', itemIndex);
|
|
48
|
+
const query = this.getNodeParameter('query', itemIndex);
|
|
49
|
+
const item = items[itemIndex].json;
|
|
50
|
+
const html = item[htmlField] || item.html || '';
|
|
51
|
+
if (!html || typeof html !== 'string') {
|
|
52
|
+
results.push({
|
|
53
|
+
json: {
|
|
54
|
+
ok: false,
|
|
55
|
+
error: `${htmlField} not found or not a string`,
|
|
56
|
+
query,
|
|
57
|
+
},
|
|
58
|
+
});
|
|
59
|
+
continue;
|
|
60
|
+
}
|
|
61
|
+
const prefetchUrl = extractPrefetchUrlFromHtml(html);
|
|
62
|
+
if (!prefetchUrl) {
|
|
63
|
+
results.push({
|
|
64
|
+
json: {
|
|
65
|
+
ok: false,
|
|
66
|
+
error: 'Could not extract prefetch URL from HTML',
|
|
67
|
+
query,
|
|
68
|
+
html_length: html.length,
|
|
69
|
+
},
|
|
70
|
+
});
|
|
71
|
+
continue;
|
|
72
|
+
}
|
|
73
|
+
results.push({
|
|
74
|
+
json: {
|
|
75
|
+
ok: true,
|
|
76
|
+
prefetch_url: prefetchUrl,
|
|
77
|
+
query,
|
|
78
|
+
base_url: prefetchUrl,
|
|
79
|
+
},
|
|
80
|
+
});
|
|
81
|
+
}
|
|
82
|
+
return [results];
|
|
83
|
+
}
|
|
84
|
+
catch (error) {
|
|
85
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
86
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Extract prefetch URL operation failed: ${msg}`);
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
//# sourceMappingURL=extractPrefetchUrl.operation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"extractPrefetchUrl.operation.js","sourceRoot":"","sources":["../../../../../nodes/GoogleMapsScraper/actions/viewport/extractPrefetchUrl.operation.ts"],"names":[],"mappings":";;;AAqDA,0BAoDC;AAzGD,+CAAuH;AAE1G,QAAA,UAAU,GAAsB;IAC3C;QACE,WAAW,EAAE,YAAY;QACzB,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,MAAM;QACf,WAAW,EAAE,+BAA+B;QAC5C,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,oBAAoB,CAAC,EAAE;SACpE;KACF;IACD;QACE,WAAW,EAAE,OAAO;QACpB,IAAI,EAAE,OAAO;QACb,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,EAAE;QACX,WAAW,EAAE,kCAAkC;QAC/C,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,oBAAoB,CAAC,EAAE;SACpE;KACF;CACF,CAAC;AAOF,SAAS,0BAA0B,CAAC,IAAY;IAE9C,MAAM,QAAQ,GAAG;QACf,oCAAoC;QACpC,8CAA8C;QAC9C,wCAAwC;KACzC,CAAC;IAEF,KAAK,MAAM,OAAO,IAAI,QAAQ,EAAE,CAAC;QAC/B,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACjC,IAAI,KAAK,IAAI,KAAK,CAAC,CAAC,CAAC,EAAE,CAAC;YACtB,OAAO,kBAAkB,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC;QACtC,CAAC;IACH,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAOM,KAAK,UAAU,OAAO;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;IAClC,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,IAAI,CAAC;QACH,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,SAAS,CAAW,CAAC;YAC1E,MAAM,KAAK,GAAG,IAAI,CAAC,gBAAgB,CAAC,OAAO,EAAE,SAAS,CAAW,CAAC;YAClE,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,IAAmB,CAAC;YAElD,MAAM,IAAI,GAAI,IAAI,CAAC,SAAS,CAAY,IAAK,IAAI,CAAC,IAAe,IAAI,EAAE,CAAC;YAExE,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,QAAQ,EAAE,CAAC;gBACtC,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE;wBACJ,EAAE,EAAE,KAAK;wBACT,KAAK,EAAE,GAAG,SAAS,4BAA4B;wBAC/C,KAAK;qBACN;iBACF,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,MAAM,WAAW,GAAG,0BAA0B,CAAC,IAAI,CAAC,CAAC;YAErD,IAAI,CAAC,WAAW,EAAE,CAAC;gBACjB,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE;wBACJ,EAAE,EAAE,KAAK;wBACT,KAAK,EAAE,0CAA0C;wBACjD,KAAK;wBACL,WAAW,EAAE,IAAI,CAAC,MAAM;qBACzB;iBACF,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAED,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE;oBACJ,EAAE,EAAE,IAAI;oBACR,YAAY,EAAE,WAAW;oBACzB,KAAK;oBACL,QAAQ,EAAE,WAAW;iBACtB;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnE,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,0CAA0C,GAAG,EAAE,CAAC,CAAC;IAChG,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.parameters = void 0;
|
|
7
|
+
exports.execute = execute;
|
|
8
|
+
const n8n_workflow_1 = require("n8n-workflow");
|
|
9
|
+
const axios_1 = __importDefault(require("axios"));
|
|
10
|
+
const proxy_1 = require("../../helpers/proxy");
|
|
11
|
+
exports.parameters = [
|
|
12
|
+
{
|
|
13
|
+
displayName: 'Search URL',
|
|
14
|
+
name: 'searchUrl',
|
|
15
|
+
type: 'string',
|
|
16
|
+
default: '',
|
|
17
|
+
required: true,
|
|
18
|
+
description: 'Google Maps search URL',
|
|
19
|
+
displayOptions: {
|
|
20
|
+
show: { resource: ['viewport'], operation: ['fetchHtmlShell'] },
|
|
21
|
+
},
|
|
22
|
+
},
|
|
23
|
+
];
|
|
24
|
+
async function execute() {
|
|
25
|
+
const items = this.getInputData();
|
|
26
|
+
const results = [];
|
|
27
|
+
try {
|
|
28
|
+
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
|
29
|
+
const searchUrl = this.getNodeParameter('searchUrl', itemIndex);
|
|
30
|
+
const proxyHost = this.getNodeParameter('proxyHost', itemIndex);
|
|
31
|
+
const proxyPort = this.getNodeParameter('proxyPort', itemIndex);
|
|
32
|
+
const proxyUsername = this.getNodeParameter('proxyUsername', itemIndex);
|
|
33
|
+
const proxyPassword = this.getNodeParameter('proxyPassword', itemIndex);
|
|
34
|
+
const proxyCountry = this.getNodeParameter('proxyCountry', itemIndex);
|
|
35
|
+
const sessionId = Math.random().toString(36).substring(2, 12);
|
|
36
|
+
const proxyAuthHeader = (0, proxy_1.buildProxyAuthHeader)(proxyUsername, proxyPassword, proxyCountry, sessionId);
|
|
37
|
+
try {
|
|
38
|
+
const response = await axios_1.default.get(searchUrl, {
|
|
39
|
+
headers: {
|
|
40
|
+
'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
41
|
+
'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
|
|
42
|
+
'accept-language': 'en-US,en;q=0.9',
|
|
43
|
+
'referer': 'https://www.google.com/',
|
|
44
|
+
'Proxy-Authorization': proxyAuthHeader,
|
|
45
|
+
},
|
|
46
|
+
proxy: (0, proxy_1.getProxyConfig)(proxyHost, proxyPort) || undefined,
|
|
47
|
+
timeout: 30000,
|
|
48
|
+
responseType: 'text',
|
|
49
|
+
});
|
|
50
|
+
results.push({
|
|
51
|
+
json: {
|
|
52
|
+
ok: true,
|
|
53
|
+
html: response.data,
|
|
54
|
+
url: searchUrl,
|
|
55
|
+
session_id: sessionId,
|
|
56
|
+
},
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
61
|
+
results.push({
|
|
62
|
+
json: {
|
|
63
|
+
ok: false,
|
|
64
|
+
error: msg,
|
|
65
|
+
url: searchUrl,
|
|
66
|
+
session_id: sessionId,
|
|
67
|
+
},
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
return [results];
|
|
72
|
+
}
|
|
73
|
+
catch (error) {
|
|
74
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
75
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Fetch HTML shell operation failed: ${msg}`);
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=fetchHtmlShell.operation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"fetchHtmlShell.operation.js","sourceRoot":"","sources":["../../../../../nodes/GoogleMapsScraper/actions/viewport/fetchHtmlShell.operation.ts"],"names":[],"mappings":";;;;;;AAyBA,0BAwDC;AAjFD,+CAA0G;AAG1G,kDAA0B;AAC1B,+CAA2E;AAE9D,QAAA,UAAU,GAAsB;IAC3C;QACE,WAAW,EAAE,YAAY;QACzB,IAAI,EAAE,WAAW;QACjB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,EAAE;QACX,QAAQ,EAAE,IAAI;QACd,WAAW,EAAE,wBAAwB;QACrC,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,gBAAgB,CAAC,EAAE;SAChE;KACF;CACF,CAAC;AAOK,KAAK,UAAU,OAAO;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;IAClC,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,IAAI,CAAC;QACH,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YAC9D,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,SAAS,CAAW,CAAC;YAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,SAAS,CAAW,CAAC;YAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,SAAS,CAAW,CAAC;YAC1E,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,SAAS,CAAW,CAAC;YAClF,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,SAAS,CAAW,CAAC;YAClF,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,SAAS,CAAW,CAAC;YAEhF,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC9D,MAAM,eAAe,GAAG,IAAA,4BAAoB,EAAC,aAAa,EAAE,aAAa,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;YAEpG,IAAI,CAAC;gBACH,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,GAAG,CAAC,SAAS,EAAE;oBAC1C,OAAO,EAAE;wBACP,YAAY,EAAE,iHAAiH;wBAC/H,QAAQ,EAAE,iEAAiE;wBAC3E,iBAAiB,EAAE,gBAAgB;wBACnC,SAAS,EAAE,yBAAyB;wBACpC,qBAAqB,EAAE,eAAe;qBACvC;oBACD,KAAK,EAAE,IAAA,sBAAc,EAAC,SAAS,EAAE,SAAS,CAAC,IAAI,SAAS;oBACxD,OAAO,EAAE,KAAK;oBACd,YAAY,EAAE,MAAM;iBACrB,CAAC,CAAC;gBAEH,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE;wBACJ,EAAE,EAAE,IAAI;wBACR,IAAI,EAAE,QAAQ,CAAC,IAAI;wBACnB,GAAG,EAAE,SAAS;wBACd,UAAU,EAAE,SAAS;qBACtB;iBACF,CAAC,CAAC;YACL,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;gBACnE,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE;wBACJ,EAAE,EAAE,KAAK;wBACT,KAAK,EAAE,GAAG;wBACV,GAAG,EAAE,SAAS;wBACd,UAAU,EAAE,SAAS;qBACtB;iBACF,CAAC,CAAC;YACL,CAAC;QACH,CAAC;QAED,OAAO,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnE,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,sCAAsC,GAAG,EAAE,CAAC,CAAC;IAC5F,CAAC;AACH,CAAC"}
|