n8n-nodes-google-map-scraper 0.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE.md +19 -0
- package/README.md +247 -0
- package/dist/icons/googlemeet.svg +4 -0
- package/dist/nodes/GoogleMapsScraper/GoogleMapsScraper.node.d.ts +5 -0
- package/dist/nodes/GoogleMapsScraper/GoogleMapsScraper.node.js +179 -0
- package/dist/nodes/GoogleMapsScraper/GoogleMapsScraper.node.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/generateGrid.operation.d.ts +18 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/generateGrid.operation.js +84 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/generateGrid.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/index.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/index.js +59 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/index.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/lookupBbox.operation.d.ts +11 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/lookupBbox.operation.js +91 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/lookupBbox.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/shrinkBbox.operation.d.ts +11 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/shrinkBbox.operation.js +82 -0
- package/dist/nodes/GoogleMapsScraper/actions/city/shrinkBbox.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/index.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/index.js +54 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/index.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/multiViewportGrid.operation.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/multiViewportGrid.operation.js +78 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/multiViewportGrid.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/singleViewport.operation.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/singleViewport.operation.js +89 -0
- package/dist/nodes/GoogleMapsScraper/actions/scrape/singleViewport.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/extractPrefetchUrl.operation.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/extractPrefetchUrl.operation.js +89 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/extractPrefetchUrl.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/fetchHtmlShell.operation.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/fetchHtmlShell.operation.js +78 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/fetchHtmlShell.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/index.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/index.js +64 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/index.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/paginateResults.operation.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/paginateResults.operation.js +246 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/paginateResults.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/parseRecords.operation.d.ts +3 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/parseRecords.operation.js +72 -0
- package/dist/nodes/GoogleMapsScraper/actions/viewport/parseRecords.operation.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/googleMapsScraper.svg +5 -0
- package/dist/nodes/GoogleMapsScraper/helpers/pagination.d.ts +5 -0
- package/dist/nodes/GoogleMapsScraper/helpers/pagination.js +29 -0
- package/dist/nodes/GoogleMapsScraper/helpers/pagination.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/helpers/parser.d.ts +18 -0
- package/dist/nodes/GoogleMapsScraper/helpers/parser.js +78 -0
- package/dist/nodes/GoogleMapsScraper/helpers/parser.js.map +1 -0
- package/dist/nodes/GoogleMapsScraper/helpers/proxy.d.ts +6 -0
- package/dist/nodes/GoogleMapsScraper/helpers/proxy.js +20 -0
- package/dist/nodes/GoogleMapsScraper/helpers/proxy.js.map +1 -0
- package/dist/package.json +54 -0
- package/dist/tsconfig.tsbuildinfo +1 -0
- package/package.json +54 -0
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
|
|
3
|
+
if (k2 === undefined) k2 = k;
|
|
4
|
+
var desc = Object.getOwnPropertyDescriptor(m, k);
|
|
5
|
+
if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
|
|
6
|
+
desc = { enumerable: true, get: function() { return m[k]; } };
|
|
7
|
+
}
|
|
8
|
+
Object.defineProperty(o, k2, desc);
|
|
9
|
+
}) : (function(o, m, k, k2) {
|
|
10
|
+
if (k2 === undefined) k2 = k;
|
|
11
|
+
o[k2] = m[k];
|
|
12
|
+
}));
|
|
13
|
+
var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
|
|
14
|
+
Object.defineProperty(o, "default", { enumerable: true, value: v });
|
|
15
|
+
}) : function(o, v) {
|
|
16
|
+
o["default"] = v;
|
|
17
|
+
});
|
|
18
|
+
var __importStar = (this && this.__importStar) || (function () {
|
|
19
|
+
var ownKeys = function(o) {
|
|
20
|
+
ownKeys = Object.getOwnPropertyNames || function (o) {
|
|
21
|
+
var ar = [];
|
|
22
|
+
for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
|
|
23
|
+
return ar;
|
|
24
|
+
};
|
|
25
|
+
return ownKeys(o);
|
|
26
|
+
};
|
|
27
|
+
return function (mod) {
|
|
28
|
+
if (mod && mod.__esModule) return mod;
|
|
29
|
+
var result = {};
|
|
30
|
+
if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
|
|
31
|
+
__setModuleDefault(result, mod);
|
|
32
|
+
return result;
|
|
33
|
+
};
|
|
34
|
+
})();
|
|
35
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
36
|
+
exports.parameters = void 0;
|
|
37
|
+
exports.execute = execute;
|
|
38
|
+
const n8n_workflow_1 = require("n8n-workflow");
|
|
39
|
+
const fetchHtmlShell = __importStar(require("./fetchHtmlShell.operation"));
|
|
40
|
+
const extractPrefetchUrl = __importStar(require("./extractPrefetchUrl.operation"));
|
|
41
|
+
const paginateResults = __importStar(require("./paginateResults.operation"));
|
|
42
|
+
const parseRecords = __importStar(require("./parseRecords.operation"));
|
|
43
|
+
exports.parameters = [
|
|
44
|
+
...fetchHtmlShell.parameters,
|
|
45
|
+
...extractPrefetchUrl.parameters,
|
|
46
|
+
...paginateResults.parameters,
|
|
47
|
+
...parseRecords.parameters,
|
|
48
|
+
];
|
|
49
|
+
async function execute(operation) {
|
|
50
|
+
if (operation === 'fetchHtmlShell') {
|
|
51
|
+
return fetchHtmlShell.execute.call(this);
|
|
52
|
+
}
|
|
53
|
+
if (operation === 'extractPrefetchUrl') {
|
|
54
|
+
return extractPrefetchUrl.execute.call(this);
|
|
55
|
+
}
|
|
56
|
+
if (operation === 'paginateResults') {
|
|
57
|
+
return paginateResults.execute.call(this);
|
|
58
|
+
}
|
|
59
|
+
if (operation === 'parseRecords') {
|
|
60
|
+
return parseRecords.execute.call(this);
|
|
61
|
+
}
|
|
62
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Unknown viewport operation: ${operation}`);
|
|
63
|
+
}
|
|
64
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"index.js","sourceRoot":"","sources":["../../../../../nodes/GoogleMapsScraper/actions/viewport/index.ts"],"names":[],"mappings":";;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAaA,0BAiBC;AA9BD,+CAA0G;AAC1G,2EAA6D;AAC7D,mFAAqE;AACrE,6EAA+D;AAC/D,uEAAyD;AAE5C,QAAA,UAAU,GAAsB;IAC3C,GAAG,cAAc,CAAC,UAAU;IAC5B,GAAG,kBAAkB,CAAC,UAAU;IAChC,GAAG,eAAe,CAAC,UAAU;IAC7B,GAAG,YAAY,CAAC,UAAU;CAC3B,CAAC;AAEK,KAAK,UAAU,OAAO,CAE3B,SAAiB;IAEjB,IAAI,SAAS,KAAK,gBAAgB,EAAE,CAAC;QACnC,OAAO,cAAc,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC3C,CAAC;IACD,IAAI,SAAS,KAAK,oBAAoB,EAAE,CAAC;QACvC,OAAO,kBAAkB,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC/C,CAAC;IACD,IAAI,SAAS,KAAK,iBAAiB,EAAE,CAAC;QACpC,OAAO,eAAe,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC5C,CAAC;IACD,IAAI,SAAS,KAAK,cAAc,EAAE,CAAC;QACjC,OAAO,YAAY,CAAC,OAAO,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACzC,CAAC;IACD,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,+BAA+B,SAAS,EAAE,CAAC,CAAC;AAC3F,CAAC"}
|
|
@@ -0,0 +1,246 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
var __importDefault = (this && this.__importDefault) || function (mod) {
|
|
3
|
+
return (mod && mod.__esModule) ? mod : { "default": mod };
|
|
4
|
+
};
|
|
5
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
6
|
+
exports.parameters = void 0;
|
|
7
|
+
exports.execute = execute;
|
|
8
|
+
const n8n_workflow_1 = require("n8n-workflow");
|
|
9
|
+
const axios_1 = __importDefault(require("axios"));
|
|
10
|
+
const proxy_1 = require("../../helpers/proxy");
|
|
11
|
+
const parser_1 = require("../../helpers/parser");
|
|
12
|
+
exports.parameters = [
|
|
13
|
+
{
|
|
14
|
+
displayName: 'Base URL',
|
|
15
|
+
name: 'baseUrl',
|
|
16
|
+
type: 'string',
|
|
17
|
+
default: '',
|
|
18
|
+
required: true,
|
|
19
|
+
description: 'Google Maps search URL from prefetch extraction',
|
|
20
|
+
displayOptions: {
|
|
21
|
+
show: { resource: ['viewport'], operation: ['paginateResults'] },
|
|
22
|
+
},
|
|
23
|
+
},
|
|
24
|
+
{
|
|
25
|
+
displayName: 'Max Results',
|
|
26
|
+
name: 'maxResults',
|
|
27
|
+
type: 'number',
|
|
28
|
+
default: 240,
|
|
29
|
+
displayOptions: {
|
|
30
|
+
show: { resource: ['viewport'], operation: ['paginateResults'] },
|
|
31
|
+
},
|
|
32
|
+
},
|
|
33
|
+
{
|
|
34
|
+
displayName: 'Cell ID',
|
|
35
|
+
name: 'cell',
|
|
36
|
+
type: 'string',
|
|
37
|
+
default: 'auto',
|
|
38
|
+
displayOptions: {
|
|
39
|
+
show: { resource: ['viewport'], operation: ['paginateResults'] },
|
|
40
|
+
},
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
displayName: 'Latitude',
|
|
44
|
+
name: 'lat',
|
|
45
|
+
type: 'number',
|
|
46
|
+
default: 0,
|
|
47
|
+
displayOptions: {
|
|
48
|
+
show: { resource: ['viewport'], operation: ['paginateResults'] },
|
|
49
|
+
},
|
|
50
|
+
},
|
|
51
|
+
{
|
|
52
|
+
displayName: 'Longitude',
|
|
53
|
+
name: 'lng',
|
|
54
|
+
type: 'number',
|
|
55
|
+
default: 0,
|
|
56
|
+
displayOptions: {
|
|
57
|
+
show: { resource: ['viewport'], operation: ['paginateResults'] },
|
|
58
|
+
},
|
|
59
|
+
},
|
|
60
|
+
{
|
|
61
|
+
displayName: 'Snapshot ID',
|
|
62
|
+
name: 'snapshotId',
|
|
63
|
+
type: 'string',
|
|
64
|
+
default: '',
|
|
65
|
+
displayOptions: {
|
|
66
|
+
show: { resource: ['viewport'], operation: ['paginateResults'] },
|
|
67
|
+
},
|
|
68
|
+
},
|
|
69
|
+
];
|
|
70
|
+
const DELAY_MS = 1500;
|
|
71
|
+
const MIN_RESPONSE_SIZE = 10000;
|
|
72
|
+
const MAX_OFFSET = 400;
|
|
73
|
+
const PAGE_SIZE = 20;
|
|
74
|
+
const COOKIES = 'CONSENT=YES+cb.20210720-07-p0.en+FX+410; SOCS=CAESHAgBEhJnd3NfMjAyMzA1MTYtMF9SQzIaAmVuIAEaBgiAjYWkBg';
|
|
75
|
+
const USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36';
|
|
76
|
+
function sleep(ms) {
|
|
77
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
78
|
+
}
|
|
79
|
+
async function fetchPage(url, proxyAuthHeader, proxyConfig) {
|
|
80
|
+
const config = {
|
|
81
|
+
headers: {
|
|
82
|
+
'user-agent': USER_AGENT,
|
|
83
|
+
'accept': '*/*',
|
|
84
|
+
'accept-language': 'en-US,en;q=0.9',
|
|
85
|
+
'referer': 'https://www.google.com/maps/',
|
|
86
|
+
'cookie': COOKIES,
|
|
87
|
+
'Proxy-Authorization': proxyAuthHeader,
|
|
88
|
+
},
|
|
89
|
+
proxy: (0, proxy_1.getProxyConfig)(proxyConfig.host, proxyConfig.port) || undefined,
|
|
90
|
+
timeout: 30000,
|
|
91
|
+
responseType: 'text',
|
|
92
|
+
transformResponse: [(data) => data],
|
|
93
|
+
validateStatus: (status) => status >= 200 && status < 500,
|
|
94
|
+
};
|
|
95
|
+
try {
|
|
96
|
+
const response = await axios_1.default.get(url, config);
|
|
97
|
+
return response.data;
|
|
98
|
+
}
|
|
99
|
+
catch (error) {
|
|
100
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
101
|
+
throw new Error(`Fetch failed: ${msg}`);
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
async function execute() {
|
|
105
|
+
const items = this.getInputData();
|
|
106
|
+
const results = [];
|
|
107
|
+
try {
|
|
108
|
+
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
|
109
|
+
const baseUrl = this.getNodeParameter('baseUrl', itemIndex);
|
|
110
|
+
const maxResults = this.getNodeParameter('maxResults', itemIndex);
|
|
111
|
+
const cell = this.getNodeParameter('cell', itemIndex);
|
|
112
|
+
const lat = this.getNodeParameter('lat', itemIndex);
|
|
113
|
+
const lng = this.getNodeParameter('lng', itemIndex);
|
|
114
|
+
const snapshotId = this.getNodeParameter('snapshotId', itemIndex);
|
|
115
|
+
const proxyHost = this.getNodeParameter('proxyHost', itemIndex);
|
|
116
|
+
const proxyPort = this.getNodeParameter('proxyPort', itemIndex);
|
|
117
|
+
const proxyUsername = this.getNodeParameter('proxyUsername', itemIndex);
|
|
118
|
+
const proxyPassword = this.getNodeParameter('proxyPassword', itemIndex);
|
|
119
|
+
const proxyCountry = this.getNodeParameter('proxyCountry', itemIndex);
|
|
120
|
+
const sessionId = Math.random().toString(36).substring(2, 12);
|
|
121
|
+
const proxyConfig = {
|
|
122
|
+
host: proxyHost,
|
|
123
|
+
port: proxyPort,
|
|
124
|
+
username: proxyUsername,
|
|
125
|
+
password: proxyPassword,
|
|
126
|
+
country: proxyCountry,
|
|
127
|
+
};
|
|
128
|
+
const proxyAuthHeader = (0, proxy_1.buildProxyAuthHeader)(proxyUsername, proxyPassword, proxyCountry, sessionId);
|
|
129
|
+
const context = {
|
|
130
|
+
baseUrl,
|
|
131
|
+
maxResults,
|
|
132
|
+
evomiSessionId: sessionId,
|
|
133
|
+
cell,
|
|
134
|
+
lat,
|
|
135
|
+
lng,
|
|
136
|
+
snapshotId,
|
|
137
|
+
proxyConfig,
|
|
138
|
+
};
|
|
139
|
+
const result = await paginateAndParse(context, proxyAuthHeader);
|
|
140
|
+
results.push({ json: result });
|
|
141
|
+
}
|
|
142
|
+
return [results];
|
|
143
|
+
}
|
|
144
|
+
catch (error) {
|
|
145
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
146
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Pagination operation failed: ${msg}`);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
async function paginateAndParse(ctx, proxyAuthHeader) {
|
|
150
|
+
const startTime = Date.now();
|
|
151
|
+
const allPlaces = new Map();
|
|
152
|
+
let lastOffset = 0;
|
|
153
|
+
let stoppedReason = 'unknown';
|
|
154
|
+
try {
|
|
155
|
+
for (let offset = 0; offset <= MAX_OFFSET; offset += PAGE_SIZE) {
|
|
156
|
+
if (allPlaces.size >= ctx.maxResults) {
|
|
157
|
+
stoppedReason = 'max_results_reached';
|
|
158
|
+
break;
|
|
159
|
+
}
|
|
160
|
+
let pageUrl = ctx.baseUrl;
|
|
161
|
+
if (offset > 0) {
|
|
162
|
+
pageUrl = ctx.baseUrl.replace('%217i20%2110b1', `%217i20%218i${offset}%2110b1`);
|
|
163
|
+
if (pageUrl === ctx.baseUrl) {
|
|
164
|
+
stoppedReason = 'pagination_param_not_found';
|
|
165
|
+
break;
|
|
166
|
+
}
|
|
167
|
+
}
|
|
168
|
+
let responseText;
|
|
169
|
+
try {
|
|
170
|
+
responseText = await fetchPage(pageUrl, proxyAuthHeader, ctx.proxyConfig);
|
|
171
|
+
}
|
|
172
|
+
catch (error) {
|
|
173
|
+
stoppedReason = `fetch_error: ${error instanceof Error ? error.message : String(error)}`;
|
|
174
|
+
break;
|
|
175
|
+
}
|
|
176
|
+
if (responseText.length < MIN_RESPONSE_SIZE) {
|
|
177
|
+
stoppedReason = 'cap_hit';
|
|
178
|
+
break;
|
|
179
|
+
}
|
|
180
|
+
const parsed = (0, parser_1.parseResponse)(responseText);
|
|
181
|
+
if (parsed.length === 0) {
|
|
182
|
+
stoppedReason = 'empty_parse';
|
|
183
|
+
break;
|
|
184
|
+
}
|
|
185
|
+
let newCount = 0;
|
|
186
|
+
for (const place of parsed) {
|
|
187
|
+
if (!allPlaces.has(place.fid)) {
|
|
188
|
+
allPlaces.set(place.fid, place);
|
|
189
|
+
newCount++;
|
|
190
|
+
}
|
|
191
|
+
if (allPlaces.size >= ctx.maxResults) {
|
|
192
|
+
break;
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
responseText = '';
|
|
196
|
+
if (newCount === 0 && offset > 0) {
|
|
197
|
+
stoppedReason = 'all_duplicates';
|
|
198
|
+
break;
|
|
199
|
+
}
|
|
200
|
+
lastOffset = offset;
|
|
201
|
+
if (offset < MAX_OFFSET) {
|
|
202
|
+
await sleep(DELAY_MS);
|
|
203
|
+
}
|
|
204
|
+
}
|
|
205
|
+
const places = Array.from(allPlaces.values()).slice(0, ctx.maxResults);
|
|
206
|
+
const totalSeconds = parseFloat(((Date.now() - startTime) / 1000).toFixed(1));
|
|
207
|
+
const result = {
|
|
208
|
+
snapshotId: ctx.snapshotId,
|
|
209
|
+
cell: ctx.cell,
|
|
210
|
+
lat: ctx.lat,
|
|
211
|
+
lng: ctx.lng,
|
|
212
|
+
ok: true,
|
|
213
|
+
places_count: places.length,
|
|
214
|
+
places: places,
|
|
215
|
+
stats: {
|
|
216
|
+
raw_count: places.length,
|
|
217
|
+
last_offset: lastOffset,
|
|
218
|
+
stop_reason: stoppedReason,
|
|
219
|
+
seconds: totalSeconds,
|
|
220
|
+
proxy_session: ctx.evomiSessionId,
|
|
221
|
+
},
|
|
222
|
+
};
|
|
223
|
+
return result;
|
|
224
|
+
}
|
|
225
|
+
catch (error) {
|
|
226
|
+
const totalSeconds = parseFloat(((Date.now() - startTime) / 1000).toFixed(1));
|
|
227
|
+
const result = {
|
|
228
|
+
snapshotId: ctx.snapshotId,
|
|
229
|
+
cell: ctx.cell,
|
|
230
|
+
lat: ctx.lat,
|
|
231
|
+
lng: ctx.lng,
|
|
232
|
+
ok: false,
|
|
233
|
+
places_count: 0,
|
|
234
|
+
places: [],
|
|
235
|
+
stats: {
|
|
236
|
+
raw_count: 0,
|
|
237
|
+
last_offset: lastOffset,
|
|
238
|
+
stop_reason: `crash: ${error instanceof Error ? error.message : String(error) || 'unknown'}`,
|
|
239
|
+
seconds: totalSeconds,
|
|
240
|
+
proxy_session: ctx.evomiSessionId,
|
|
241
|
+
},
|
|
242
|
+
};
|
|
243
|
+
return result;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
//# sourceMappingURL=paginateResults.operation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"paginateResults.operation.js","sourceRoot":"","sources":["../../../../../nodes/GoogleMapsScraper/actions/viewport/paginateResults.operation.ts"],"names":[],"mappings":";;;;;;AA2JA,0BAkDC;AA7MD,+CAAuH;AAGvH,kDAAkD;AAClD,+CAA2E;AAC3E,iDAAkE;AAErD,QAAA,UAAU,GAAsB;IAC3C;QACE,WAAW,EAAE,UAAU;QACvB,IAAI,EAAE,SAAS;QACf,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,EAAE;QACX,QAAQ,EAAE,IAAI;QACd,WAAW,EAAE,iDAAiD;QAC9D,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,iBAAiB,CAAC,EAAE;SACjE;KACF;IACD;QACE,WAAW,EAAE,aAAa;QAC1B,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,GAAG;QACZ,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,iBAAiB,CAAC,EAAE;SACjE;KACF;IACD;QACE,WAAW,EAAE,SAAS;QACtB,IAAI,EAAE,MAAM;QACZ,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,MAAM;QACf,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,iBAAiB,CAAC,EAAE;SACjE;KACF;IACD;QACE,WAAW,EAAE,UAAU;QACvB,IAAI,EAAE,KAAK;QACX,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,CAAC;QACV,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,iBAAiB,CAAC,EAAE;SACjE;KACF;IACD;QACE,WAAW,EAAE,WAAW;QACxB,IAAI,EAAE,KAAK;QACX,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,CAAC;QACV,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,iBAAiB,CAAC,EAAE;SACjE;KACF;IACD;QACE,WAAW,EAAE,aAAa;QAC1B,IAAI,EAAE,YAAY;QAClB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,EAAE;QACX,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,iBAAiB,CAAC,EAAE;SACjE;KACF;CACF,CAAC;AAEF,MAAM,QAAQ,GAAG,IAAI,CAAC;AACtB,MAAM,iBAAiB,GAAG,KAAK,CAAC;AAChC,MAAM,UAAU,GAAG,GAAG,CAAC;AACvB,MAAM,SAAS,GAAG,EAAE,CAAC;AAErB,MAAM,OAAO,GAAG,sGAAsG,CAAC;AACvH,MAAM,UAAU,GAAG,iHAAiH,CAAC;AAoCrI,SAAS,KAAK,CAAC,EAAU;IACvB,OAAO,IAAI,OAAO,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,UAAU,CAAC,OAAO,EAAE,EAAE,CAAC,CAAC,CAAC;AAC3D,CAAC;AAWD,KAAK,UAAU,SAAS,CACtB,GAAW,EACX,eAAuB,EACvB,WAAwB;IAExB,MAAM,MAAM,GAAuB;QACjC,OAAO,EAAE;YACP,YAAY,EAAE,UAAU;YACxB,QAAQ,EAAE,KAAK;YACf,iBAAiB,EAAE,gBAAgB;YACnC,SAAS,EAAE,8BAA8B;YACzC,QAAQ,EAAE,OAAO;YACjB,qBAAqB,EAAE,eAAe;SACvC;QACD,KAAK,EAAE,IAAA,sBAAc,EAAC,WAAW,CAAC,IAAI,EAAE,WAAW,CAAC,IAAI,CAAC,IAAI,SAAS;QACtE,OAAO,EAAE,KAAK;QACd,YAAY,EAAE,MAAM;QACpB,iBAAiB,EAAE,CAAC,CAAC,IAAa,EAAE,EAAE,CAAC,IAAI,CAAC;QAC5C,cAAc,EAAE,CAAC,MAAc,EAAE,EAAE,CAAC,MAAM,IAAI,GAAG,IAAI,MAAM,GAAG,GAAG;KAClE,CAAC;IAEF,IAAI,CAAC;QACH,MAAM,QAAQ,GAAG,MAAM,eAAK,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;QAC9C,OAAO,QAAQ,CAAC,IAAc,CAAC;IACjC,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnE,MAAM,IAAI,KAAK,CAAC,iBAAiB,GAAG,EAAE,CAAC,CAAC;IAC1C,CAAC;AACH,CAAC;AAMM,KAAK,UAAU,OAAO;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;IAClC,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,IAAI,CAAC;QACH,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YAC9D,MAAM,OAAO,GAAG,IAAI,CAAC,gBAAgB,CAAC,SAAS,EAAE,SAAS,CAAW,CAAC;YACtE,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,SAAS,CAAW,CAAC;YAC5E,MAAM,IAAI,GAAG,IAAI,CAAC,gBAAgB,CAAC,MAAM,EAAE,SAAS,CAAW,CAAC;YAChE,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAW,CAAC;YAC9D,MAAM,GAAG,GAAG,IAAI,CAAC,gBAAgB,CAAC,KAAK,EAAE,SAAS,CAAW,CAAC;YAC9D,MAAM,UAAU,GAAG,IAAI,CAAC,gBAAgB,CAAC,YAAY,EAAE,SAAS,CAAW,CAAC;YAG5E,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,SAAS,CAAW,CAAC;YAC1E,MAAM,SAAS,GAAG,IAAI,CAAC,gBAAgB,CAAC,WAAW,EAAE,SAAS,CAAW,CAAC;YAC1E,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,SAAS,CAAW,CAAC;YAClF,MAAM,aAAa,GAAG,IAAI,CAAC,gBAAgB,CAAC,eAAe,EAAE,SAAS,CAAW,CAAC;YAClF,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,SAAS,CAAW,CAAC;YAEhF,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,SAAS,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC;YAC9D,MAAM,WAAW,GAAgB;gBAC/B,IAAI,EAAE,SAAS;gBACf,IAAI,EAAE,SAAS;gBACf,QAAQ,EAAE,aAAa;gBACvB,QAAQ,EAAE,aAAa;gBACvB,OAAO,EAAE,YAAY;aACtB,CAAC;YACF,MAAM,eAAe,GAAG,IAAA,4BAAoB,EAAC,aAAa,EAAE,aAAa,EAAE,YAAY,EAAE,SAAS,CAAC,CAAC;YAEpG,MAAM,OAAO,GAAsB;gBACjC,OAAO;gBACP,UAAU;gBACV,cAAc,EAAE,SAAS;gBACzB,IAAI;gBACJ,GAAG;gBACH,GAAG;gBACH,UAAU;gBACV,WAAW;aACZ,CAAC;YAEF,MAAM,MAAM,GAAG,MAAM,gBAAgB,CAAC,OAAO,EAAE,eAAe,CAAC,CAAC;YAChE,OAAO,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,CAAC;QACjC,CAAC;QAED,OAAO,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnE,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,gCAAgC,GAAG,EAAE,CAAC,CAAC;IACtF,CAAC;AACH,CAAC;AAKD,KAAK,UAAU,gBAAgB,CAC7B,GAAsB,EACtB,eAAuB;IAEvB,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IAC7B,MAAM,SAAS,GAAG,IAAI,GAAG,EAAuB,CAAC;IACjD,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,aAAa,GAAG,SAAS,CAAC;IAE9B,IAAI,CAAC;QACH,KAAK,IAAI,MAAM,GAAG,CAAC,EAAE,MAAM,IAAI,UAAU,EAAE,MAAM,IAAI,SAAS,EAAE,CAAC;YAE/D,IAAI,SAAS,CAAC,IAAI,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;gBACrC,aAAa,GAAG,qBAAqB,CAAC;gBACtC,MAAM;YACR,CAAC;YAGD,IAAI,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC;YAC1B,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACf,OAAO,GAAG,GAAG,CAAC,OAAO,CAAC,OAAO,CAAC,gBAAgB,EAAE,eAAe,MAAM,SAAS,CAAC,CAAC;gBAChF,IAAI,OAAO,KAAK,GAAG,CAAC,OAAO,EAAE,CAAC;oBAC5B,aAAa,GAAG,4BAA4B,CAAC;oBAC7C,MAAM;gBACR,CAAC;YACH,CAAC;YAGD,IAAI,YAAoB,CAAC;YACzB,IAAI,CAAC;gBACH,YAAY,GAAG,MAAM,SAAS,CAAC,OAAO,EAAE,eAAe,EAAE,GAAG,CAAC,WAAW,CAAC,CAAC;YAC5E,CAAC;YAAC,OAAO,KAAK,EAAE,CAAC;gBACf,aAAa,GAAG,gBAAgB,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC;gBACzF,MAAM;YACR,CAAC;YAGD,IAAI,YAAY,CAAC,MAAM,GAAG,iBAAiB,EAAE,CAAC;gBAC5C,aAAa,GAAG,SAAS,CAAC;gBAC1B,MAAM;YACR,CAAC;YAGD,MAAM,MAAM,GAAG,IAAA,sBAAa,EAAC,YAAY,CAAC,CAAC;YAC3C,IAAI,MAAM,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACxB,aAAa,GAAG,aAAa,CAAC;gBAC9B,MAAM;YACR,CAAC;YAGD,IAAI,QAAQ,GAAG,CAAC,CAAC;YACjB,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;gBAC3B,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,EAAE,CAAC;oBAC9B,SAAS,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,EAAE,KAAK,CAAC,CAAC;oBAChC,QAAQ,EAAE,CAAC;gBACb,CAAC;gBACD,IAAI,SAAS,CAAC,IAAI,IAAI,GAAG,CAAC,UAAU,EAAE,CAAC;oBACrC,MAAM;gBACR,CAAC;YACH,CAAC;YAGD,YAAY,GAAG,EAAE,CAAC;YAGlB,IAAI,QAAQ,KAAK,CAAC,IAAI,MAAM,GAAG,CAAC,EAAE,CAAC;gBACjC,aAAa,GAAG,gBAAgB,CAAC;gBACjC,MAAM;YACR,CAAC;YAED,UAAU,GAAG,MAAM,CAAC;YAGpB,IAAI,MAAM,GAAG,UAAU,EAAE,CAAC;gBACxB,MAAM,KAAK,CAAC,QAAQ,CAAC,CAAC;YACxB,CAAC;QACH,CAAC;QAED,MAAM,MAAM,GAAG,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,CAAC,CAAC,KAAK,CAAC,CAAC,EAAE,GAAG,CAAC,UAAU,CAAC,CAAC;QACvE,MAAM,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAE5E,MAAM,MAAM,GAAqB;YACjC,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,EAAE,EAAE,IAAI;YACR,YAAY,EAAE,MAAM,CAAC,MAAM;YACzB,MAAM,EAAE,MAAmD;YAC7D,KAAK,EAAE;gBACL,SAAS,EAAE,MAAM,CAAC,MAAM;gBACxB,WAAW,EAAE,UAAU;gBACvB,WAAW,EAAE,aAAa;gBAC1B,OAAO,EAAE,YAAY;gBACrB,aAAa,EAAE,GAAG,CAAC,cAAc;aACnB;SACjB,CAAC;QAEF,OAAO,MAAM,CAAC;IAChB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QAGf,MAAM,YAAY,GAAG,UAAU,CAAC,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,SAAS,CAAC,GAAG,IAAI,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC;QAE9E,MAAM,MAAM,GAAqB;YAC/B,UAAU,EAAE,GAAG,CAAC,UAAU;YAC1B,IAAI,EAAE,GAAG,CAAC,IAAI;YACd,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,GAAG,EAAE,GAAG,CAAC,GAAG;YACZ,EAAE,EAAE,KAAK;YACT,YAAY,EAAE,CAAC;YACf,MAAM,EAAE,EAAE;YACV,KAAK,EAAE;gBACL,SAAS,EAAE,CAAC;gBACZ,WAAW,EAAE,UAAU;gBACvB,WAAW,EAAE,UAAU,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,IAAI,SAAS,EAAE;gBAC5F,OAAO,EAAE,YAAY;gBACrB,aAAa,EAAE,GAAG,CAAC,cAAc;aACnB;SACjB,CAAC;QAEF,OAAO,MAAM,CAAC;IAChB,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.parameters = void 0;
|
|
4
|
+
exports.execute = execute;
|
|
5
|
+
const n8n_workflow_1 = require("n8n-workflow");
|
|
6
|
+
const parser_1 = require("../../helpers/parser");
|
|
7
|
+
exports.parameters = [
|
|
8
|
+
{
|
|
9
|
+
displayName: 'Records Field',
|
|
10
|
+
name: 'recordsField',
|
|
11
|
+
type: 'string',
|
|
12
|
+
default: 'records',
|
|
13
|
+
description: 'Field name containing raw record arrays',
|
|
14
|
+
displayOptions: {
|
|
15
|
+
show: { resource: ['viewport'], operation: ['parseRecords'] },
|
|
16
|
+
},
|
|
17
|
+
},
|
|
18
|
+
];
|
|
19
|
+
async function execute() {
|
|
20
|
+
const items = this.getInputData();
|
|
21
|
+
const results = [];
|
|
22
|
+
try {
|
|
23
|
+
for (let itemIndex = 0; itemIndex < items.length; itemIndex++) {
|
|
24
|
+
const recordsField = this.getNodeParameter('recordsField', itemIndex);
|
|
25
|
+
const item = items[itemIndex].json;
|
|
26
|
+
const records = item[recordsField] || item.records || [];
|
|
27
|
+
if (!Array.isArray(records)) {
|
|
28
|
+
results.push({
|
|
29
|
+
json: {
|
|
30
|
+
ok: false,
|
|
31
|
+
error: `${recordsField} is not an array`,
|
|
32
|
+
parsed_count: 0,
|
|
33
|
+
records: [],
|
|
34
|
+
},
|
|
35
|
+
});
|
|
36
|
+
continue;
|
|
37
|
+
}
|
|
38
|
+
const parsedRecords = [];
|
|
39
|
+
const errors = [];
|
|
40
|
+
for (let i = 0; i < records.length; i++) {
|
|
41
|
+
try {
|
|
42
|
+
const record = records[i];
|
|
43
|
+
if (Array.isArray(record)) {
|
|
44
|
+
const parsed = (0, parser_1.parseRecord)(record);
|
|
45
|
+
if (parsed.fid && parsed.name) {
|
|
46
|
+
parsedRecords.push(parsed);
|
|
47
|
+
}
|
|
48
|
+
}
|
|
49
|
+
}
|
|
50
|
+
catch (error) {
|
|
51
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
52
|
+
errors.push(`Record ${i}: ${msg}`);
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
results.push({
|
|
56
|
+
json: {
|
|
57
|
+
ok: true,
|
|
58
|
+
parsed_count: parsedRecords.length,
|
|
59
|
+
records: parsedRecords,
|
|
60
|
+
errors: errors.length > 0 ? errors : undefined,
|
|
61
|
+
original_count: records.length,
|
|
62
|
+
},
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
return [results];
|
|
66
|
+
}
|
|
67
|
+
catch (error) {
|
|
68
|
+
const msg = error instanceof Error ? error.message : String(error);
|
|
69
|
+
throw new n8n_workflow_1.NodeOperationError(this.getNode(), `Parse records operation failed: ${msg}`);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
//# sourceMappingURL=parseRecords.operation.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parseRecords.operation.js","sourceRoot":"","sources":["../../../../../nodes/GoogleMapsScraper/actions/viewport/parseRecords.operation.ts"],"names":[],"mappings":";;;AAqBA,0BA4DC;AAjFD,+CAAuH;AACvH,iDAAgE;AAEnD,QAAA,UAAU,GAAsB;IAC3C;QACE,WAAW,EAAE,eAAe;QAC5B,IAAI,EAAE,cAAc;QACpB,IAAI,EAAE,QAAQ;QACd,OAAO,EAAE,SAAS;QAClB,WAAW,EAAE,yCAAyC;QACtD,cAAc,EAAE;YACd,IAAI,EAAE,EAAE,QAAQ,EAAE,CAAC,UAAU,CAAC,EAAE,SAAS,EAAE,CAAC,cAAc,CAAC,EAAE;SAC9D;KACF;CACF,CAAC;AAOK,KAAK,UAAU,OAAO;IAC3B,MAAM,KAAK,GAAG,IAAI,CAAC,YAAY,EAAE,CAAC;IAClC,MAAM,OAAO,GAAyB,EAAE,CAAC;IAEzC,IAAI,CAAC;QACH,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,SAAS,GAAG,KAAK,CAAC,MAAM,EAAE,SAAS,EAAE,EAAE,CAAC;YAC9D,MAAM,YAAY,GAAG,IAAI,CAAC,gBAAgB,CAAC,cAAc,EAAE,SAAS,CAAW,CAAC;YAChF,MAAM,IAAI,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC,IAAmB,CAAC;YAGlD,MAAM,OAAO,GAAG,IAAI,CAAC,YAAY,CAAC,IAAI,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC;YAEzD,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,EAAE,CAAC;gBAC5B,OAAO,CAAC,IAAI,CAAC;oBACX,IAAI,EAAE;wBACJ,EAAE,EAAE,KAAK;wBACT,KAAK,EAAE,GAAG,YAAY,kBAAkB;wBACxC,YAAY,EAAE,CAAC;wBACf,OAAO,EAAE,EAAE;qBACZ;iBACF,CAAC,CAAC;gBACH,SAAS;YACX,CAAC;YAGD,MAAM,aAAa,GAAkB,EAAE,CAAC;YACxC,MAAM,MAAM,GAAa,EAAE,CAAC;YAE5B,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,OAAO,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;gBACxC,IAAI,CAAC;oBACH,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;oBAC1B,IAAI,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;wBAC1B,MAAM,MAAM,GAAG,IAAA,oBAAW,EAAC,MAAM,CAAC,CAAC;wBAEnC,IAAI,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;4BAC9B,aAAa,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;wBAC7B,CAAC;oBACH,CAAC;gBACH,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;oBACnE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC,CAAC;gBACrC,CAAC;YACH,CAAC;YAED,OAAO,CAAC,IAAI,CAAC;gBACX,IAAI,EAAE;oBACJ,EAAE,EAAE,IAAI;oBACR,YAAY,EAAE,aAAa,CAAC,MAAM;oBAClC,OAAO,EAAE,aAAa;oBACtB,MAAM,EAAE,MAAM,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS;oBAC9C,cAAc,EAAE,OAAO,CAAC,MAAM;iBAC/B;aACF,CAAC,CAAC;QACL,CAAC;QAED,OAAO,CAAC,OAAO,CAAC,CAAC;IACnB,CAAC;IAAC,OAAO,KAAK,EAAE,CAAC;QACf,MAAM,GAAG,GAAG,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;QACnE,MAAM,IAAI,iCAAkB,CAAC,IAAI,CAAC,OAAO,EAAE,EAAE,mCAAmC,GAAG,EAAE,CAAC,CAAC;IACzF,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24" fill="none" stroke="currentColor" stroke-width="2" stroke-linecap="round" stroke-linejoin="round">
|
|
2
|
+
<!-- Google Maps icon placeholder -->
|
|
3
|
+
<path d="M21 10c0 7-9 13-9 13s-9-6-9-13a9 9 0 0 1 18 0z"/>
|
|
4
|
+
<circle cx="12" cy="10" r="3"/>
|
|
5
|
+
</svg>
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
export declare function calculatePages(totalItems: number, itemsPerPage: number): number;
|
|
2
|
+
export declare function getNextOffset(currentOffset: number, pageSize?: number): number;
|
|
3
|
+
export declare function getPaginatedUrl(baseUrl: string, offset: number): string;
|
|
4
|
+
export declare function hasPaginationParam(url: string): boolean;
|
|
5
|
+
export declare function getMaxIterations(maxResults: number, pageSize?: number, maxOffset?: number): number;
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.calculatePages = calculatePages;
|
|
4
|
+
exports.getNextOffset = getNextOffset;
|
|
5
|
+
exports.getPaginatedUrl = getPaginatedUrl;
|
|
6
|
+
exports.hasPaginationParam = hasPaginationParam;
|
|
7
|
+
exports.getMaxIterations = getMaxIterations;
|
|
8
|
+
function calculatePages(totalItems, itemsPerPage) {
|
|
9
|
+
return Math.ceil(totalItems / itemsPerPage);
|
|
10
|
+
}
|
|
11
|
+
function getNextOffset(currentOffset, pageSize = 20) {
|
|
12
|
+
return currentOffset + pageSize;
|
|
13
|
+
}
|
|
14
|
+
function getPaginatedUrl(baseUrl, offset) {
|
|
15
|
+
if (offset === 0) {
|
|
16
|
+
return baseUrl;
|
|
17
|
+
}
|
|
18
|
+
const paginatedUrl = baseUrl.replace('%217i20%2110b1', `%217i20%218i${offset}%2110b1`);
|
|
19
|
+
return paginatedUrl;
|
|
20
|
+
}
|
|
21
|
+
function hasPaginationParam(url) {
|
|
22
|
+
return url.includes('%217i20%2110b1');
|
|
23
|
+
}
|
|
24
|
+
function getMaxIterations(maxResults, pageSize = 20, maxOffset = 400) {
|
|
25
|
+
const iterationsForResults = Math.ceil(maxResults / pageSize);
|
|
26
|
+
const iterationsForOffset = Math.floor(maxOffset / pageSize);
|
|
27
|
+
return Math.min(iterationsForResults, iterationsForOffset);
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=pagination.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"pagination.js","sourceRoot":"","sources":["../../../../nodes/GoogleMapsScraper/helpers/pagination.ts"],"names":[],"mappings":";;AAWA,wCAEC;AASD,sCAEC;AASD,0CAMC;AAOD,gDAEC;AASD,4CAQC;AAtDD,SAAgB,cAAc,CAAC,UAAkB,EAAE,YAAoB;IACrE,OAAO,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,YAAY,CAAC,CAAC;AAC9C,CAAC;AASD,SAAgB,aAAa,CAAC,aAAqB,EAAE,WAAmB,EAAE;IACxE,OAAO,aAAa,GAAG,QAAQ,CAAC;AAClC,CAAC;AASD,SAAgB,eAAe,CAAC,OAAe,EAAE,MAAc;IAC7D,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;QACjB,OAAO,OAAO,CAAC;IACjB,CAAC;IACD,MAAM,YAAY,GAAG,OAAO,CAAC,OAAO,CAAC,gBAAgB,EAAE,eAAe,MAAM,SAAS,CAAC,CAAC;IACvF,OAAO,YAAY,CAAC;AACtB,CAAC;AAOD,SAAgB,kBAAkB,CAAC,GAAW;IAC5C,OAAO,GAAG,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;AACxC,CAAC;AASD,SAAgB,gBAAgB,CAC9B,UAAkB,EAClB,WAAmB,EAAE,EACrB,YAAoB,GAAG;IAEvB,MAAM,oBAAoB,GAAG,IAAI,CAAC,IAAI,CAAC,UAAU,GAAG,QAAQ,CAAC,CAAC;IAC9D,MAAM,mBAAmB,GAAG,IAAI,CAAC,KAAK,CAAC,SAAS,GAAG,QAAQ,CAAC,CAAC;IAC7D,OAAO,IAAI,CAAC,GAAG,CAAC,oBAAoB,EAAE,mBAAmB,CAAC,CAAC;AAC7D,CAAC"}
|
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
export interface PlaceRecord {
|
|
2
|
+
fid: string;
|
|
3
|
+
name: string;
|
|
4
|
+
full_address: string | null;
|
|
5
|
+
locality: string | null;
|
|
6
|
+
latitude: number | null;
|
|
7
|
+
longitude: number | null;
|
|
8
|
+
rating: number | null;
|
|
9
|
+
review_count: number | null;
|
|
10
|
+
categories: string[] | null;
|
|
11
|
+
website: string | null;
|
|
12
|
+
website_domain: string | null;
|
|
13
|
+
phone: string | null;
|
|
14
|
+
place_id_cid: string | null;
|
|
15
|
+
}
|
|
16
|
+
export declare function stripXSSI(text: string): string;
|
|
17
|
+
export declare function parseRecord(record: any[]): PlaceRecord;
|
|
18
|
+
export declare function parseResponse(responseText: string): PlaceRecord[];
|
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.stripXSSI = stripXSSI;
|
|
4
|
+
exports.parseRecord = parseRecord;
|
|
5
|
+
exports.parseResponse = parseResponse;
|
|
6
|
+
function dig(obj, ...path) {
|
|
7
|
+
let current = obj;
|
|
8
|
+
for (const p of path) {
|
|
9
|
+
if (current == null)
|
|
10
|
+
return null;
|
|
11
|
+
current = current[p];
|
|
12
|
+
}
|
|
13
|
+
return current !== null && current !== void 0 ? current : null;
|
|
14
|
+
}
|
|
15
|
+
function stripXSSI(text) {
|
|
16
|
+
return text
|
|
17
|
+
.replace(/^\/\*""\*\//, '')
|
|
18
|
+
.replace(/^\)\]\}'[\r\n]*/, '')
|
|
19
|
+
.trim();
|
|
20
|
+
}
|
|
21
|
+
function extractPhone(record) {
|
|
22
|
+
const recordStr = JSON.stringify(record);
|
|
23
|
+
const intlMatch = recordStr.match(/"\+91[\s\d]{9,14}"/);
|
|
24
|
+
if (intlMatch) {
|
|
25
|
+
return intlMatch[0].replace(/"/g, '');
|
|
26
|
+
}
|
|
27
|
+
const localMatch = recordStr.match(/"0\d{2,4}[\s\d]{6,10}"/);
|
|
28
|
+
if (localMatch) {
|
|
29
|
+
return localMatch[0].replace(/"/g, '');
|
|
30
|
+
}
|
|
31
|
+
return null;
|
|
32
|
+
}
|
|
33
|
+
function extractPlaceId(record) {
|
|
34
|
+
const recordStr = JSON.stringify(record);
|
|
35
|
+
const match = recordStr.match(/"(ChIJ[A-Za-z0-9_-]{20,})"/);
|
|
36
|
+
return match ? match[1] : null;
|
|
37
|
+
}
|
|
38
|
+
function parseRecord(record) {
|
|
39
|
+
var _a;
|
|
40
|
+
return {
|
|
41
|
+
fid: dig(record, 10),
|
|
42
|
+
name: dig(record, 11),
|
|
43
|
+
full_address: (_a = dig(record, 18)) !== null && _a !== void 0 ? _a : dig(record, 39),
|
|
44
|
+
locality: dig(record, 14),
|
|
45
|
+
latitude: dig(record, 9, 2),
|
|
46
|
+
longitude: dig(record, 9, 3),
|
|
47
|
+
rating: dig(record, 4, 7),
|
|
48
|
+
review_count: dig(record, 37, 1),
|
|
49
|
+
categories: dig(record, 13),
|
|
50
|
+
website: dig(record, 7, 0),
|
|
51
|
+
website_domain: dig(record, 7, 1),
|
|
52
|
+
phone: extractPhone(record),
|
|
53
|
+
place_id_cid: extractPlaceId(record),
|
|
54
|
+
};
|
|
55
|
+
}
|
|
56
|
+
function parseResponse(responseText) {
|
|
57
|
+
try {
|
|
58
|
+
const cleanedJson = stripXSSI(responseText);
|
|
59
|
+
const data = JSON.parse(cleanedJson);
|
|
60
|
+
const containers = dig(data, 64) || [];
|
|
61
|
+
const results = [];
|
|
62
|
+
for (const container of containers) {
|
|
63
|
+
const record = container === null || container === void 0 ? void 0 : container[1];
|
|
64
|
+
if (!Array.isArray(record) || record.length < 100) {
|
|
65
|
+
continue;
|
|
66
|
+
}
|
|
67
|
+
const parsed = parseRecord(record);
|
|
68
|
+
if (parsed.fid && parsed.name) {
|
|
69
|
+
results.push(parsed);
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
return results;
|
|
73
|
+
}
|
|
74
|
+
catch {
|
|
75
|
+
return [];
|
|
76
|
+
}
|
|
77
|
+
}
|
|
78
|
+
//# sourceMappingURL=parser.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"parser.js","sourceRoot":"","sources":["../../../../nodes/GoogleMapsScraper/helpers/parser.ts"],"names":[],"mappings":";;AA4CA,8BAKC;AA0CD,kCAgBC;AAQD,sCA4BC;AAlHD,SAAS,GAAG,CAAC,GAAQ,EAAE,GAAG,IAAW;IACnC,IAAI,OAAO,GAAG,GAAG,CAAC;IAClB,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;QACrB,IAAI,OAAO,IAAI,IAAI;YAAE,OAAO,IAAI,CAAC;QACjC,OAAO,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IACvB,CAAC;IACD,OAAO,OAAO,aAAP,OAAO,cAAP,OAAO,GAAI,IAAI,CAAC;AACzB,CAAC;AAQD,SAAgB,SAAS,CAAC,IAAY;IACpC,OAAO,IAAI;SACR,OAAO,CAAC,aAAa,EAAE,EAAE,CAAC;SAC1B,OAAO,CAAC,iBAAiB,EAAE,EAAE,CAAC;SAC9B,IAAI,EAAE,CAAC;AACZ,CAAC;AAOD,SAAS,YAAY,CAAC,MAAW;IAC/B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IAGzC,MAAM,SAAS,GAAG,SAAS,CAAC,KAAK,CAAC,oBAAoB,CAAC,CAAC;IACxD,IAAI,SAAS,EAAE,CAAC;QACd,OAAO,SAAS,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACxC,CAAC;IAGD,MAAM,UAAU,GAAG,SAAS,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC;IAC7D,IAAI,UAAU,EAAE,CAAC;QACf,OAAO,UAAU,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC;IACzC,CAAC;IAED,OAAO,IAAI,CAAC;AACd,CAAC;AAOD,SAAS,cAAc,CAAC,MAAW;IACjC,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC;IACzC,MAAM,KAAK,GAAG,SAAS,CAAC,KAAK,CAAC,4BAA4B,CAAC,CAAC;IAC5D,OAAO,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACjC,CAAC;AAQD,SAAgB,WAAW,CAAC,MAAa;;IACvC,OAAO;QACL,GAAG,EAAE,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC;QACpB,IAAI,EAAE,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC;QACrB,YAAY,EAAE,MAAA,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC,mCAAI,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC;QAChD,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC;QACzB,QAAQ,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;QAC3B,SAAS,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;QAC5B,MAAM,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;QACzB,YAAY,EAAE,GAAG,CAAC,MAAM,EAAE,EAAE,EAAE,CAAC,CAAC;QAChC,UAAU,EAAE,GAAG,CAAC,MAAM,EAAE,EAAE,CAAC;QAC3B,OAAO,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;QAC1B,cAAc,EAAE,GAAG,CAAC,MAAM,EAAE,CAAC,EAAE,CAAC,CAAC;QACjC,KAAK,EAAE,YAAY,CAAC,MAAM,CAAC;QAC3B,YAAY,EAAE,cAAc,CAAC,MAAM,CAAC;KACrC,CAAC;AACJ,CAAC;AAQD,SAAgB,aAAa,CAAC,YAAoB;IAChD,IAAI,CAAC;QACH,MAAM,WAAW,GAAG,SAAS,CAAC,YAAY,CAAC,CAAC;QAC5C,MAAM,IAAI,GAAG,IAAI,CAAC,KAAK,CAAC,WAAW,CAAC,CAAC;QAGrC,MAAM,UAAU,GAAG,GAAG,CAAC,IAAI,EAAE,EAAE,CAAC,IAAI,EAAE,CAAC;QACvC,MAAM,OAAO,GAAkB,EAAE,CAAC;QAElC,KAAK,MAAM,SAAS,IAAI,UAAU,EAAE,CAAC;YACnC,MAAM,MAAM,GAAG,SAAS,aAAT,SAAS,uBAAT,SAAS,CAAG,CAAC,CAAC,CAAC;YAG9B,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,GAAG,GAAG,EAAE,CAAC;gBAClD,SAAS;YACX,CAAC;YAGD,MAAM,MAAM,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC;YACnC,IAAI,MAAM,CAAC,GAAG,IAAI,MAAM,CAAC,IAAI,EAAE,CAAC;gBAC9B,OAAO,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;YACvB,CAAC;QACH,CAAC;QAED,OAAO,OAAO,CAAC;IACjB,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,EAAE,CAAC;IACZ,CAAC;AACH,CAAC"}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
"use strict";
|
|
2
|
+
Object.defineProperty(exports, "__esModule", { value: true });
|
|
3
|
+
exports.buildProxyAuthHeader = buildProxyAuthHeader;
|
|
4
|
+
exports.getProxyConfig = getProxyConfig;
|
|
5
|
+
function buildProxyAuthHeader(username, password, country, sessionId) {
|
|
6
|
+
const proxyPassword = `${password}_country-${country}_session-${sessionId}`;
|
|
7
|
+
const authString = `${username}:${proxyPassword}`;
|
|
8
|
+
return `Basic ${Buffer.from(authString).toString('base64')}`;
|
|
9
|
+
}
|
|
10
|
+
function getProxyConfig(host, port) {
|
|
11
|
+
if (!host) {
|
|
12
|
+
return false;
|
|
13
|
+
}
|
|
14
|
+
return {
|
|
15
|
+
protocol: 'http',
|
|
16
|
+
host,
|
|
17
|
+
port: port !== null && port !== void 0 ? port : 80,
|
|
18
|
+
};
|
|
19
|
+
}
|
|
20
|
+
//# sourceMappingURL=proxy.js.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"proxy.js","sourceRoot":"","sources":["../../../../nodes/GoogleMapsScraper/helpers/proxy.ts"],"names":[],"mappings":";;AAaA,oDASC;AAQD,wCAUC;AA3BD,SAAgB,oBAAoB,CAClC,QAAgB,EAChB,QAAgB,EAChB,OAAe,EACf,SAAiB;IAEjB,MAAM,aAAa,GAAG,GAAG,QAAQ,YAAY,OAAO,YAAY,SAAS,EAAE,CAAC;IAC5E,MAAM,UAAU,GAAG,GAAG,QAAQ,IAAI,aAAa,EAAE,CAAC;IAClD,OAAO,SAAS,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,CAAC;AAC/D,CAAC;AAQD,SAAgB,cAAc,CAAC,IAAa,EAAE,IAAa;IACzD,IAAI,CAAC,IAAI,EAAE,CAAC;QACV,OAAO,KAAK,CAAC;IACf,CAAC;IAED,OAAO;QACL,QAAQ,EAAE,MAAM;QAChB,IAAI;QACJ,IAAI,EAAE,IAAI,aAAJ,IAAI,cAAJ,IAAI,GAAI,EAAE;KACjB,CAAC;AACJ,CAAC"}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "n8n-nodes-google-map-scraper",
|
|
3
|
+
"version": "0.1.1",
|
|
4
|
+
"description": "n8n nodes for Google Maps Scraper",
|
|
5
|
+
"license": "MIT",
|
|
6
|
+
"homepage": "https://github.com/tom-cruise10/GoogleMapsScraper",
|
|
7
|
+
"keywords": [
|
|
8
|
+
"n8n-community-node-package"
|
|
9
|
+
],
|
|
10
|
+
"author": {
|
|
11
|
+
"name": "tom-cruise10",
|
|
12
|
+
"email": "tom-cruise10@gmail.com"
|
|
13
|
+
},
|
|
14
|
+
"repository": {
|
|
15
|
+
"type": "git",
|
|
16
|
+
"url": "https://github.com/tom-cruise10/GoogleMapsScraper.git"
|
|
17
|
+
},
|
|
18
|
+
"scripts": {
|
|
19
|
+
"build": "n8n-node build",
|
|
20
|
+
"build:watch": "tsc --watch",
|
|
21
|
+
"dev": "n8n-node dev",
|
|
22
|
+
"lint": "n8n-node lint",
|
|
23
|
+
"lint:fix": "n8n-node lint --fix",
|
|
24
|
+
"release": "n8n-node release",
|
|
25
|
+
"prepublishOnly": "n8n-node prerelease"
|
|
26
|
+
},
|
|
27
|
+
"files": [
|
|
28
|
+
"dist"
|
|
29
|
+
],
|
|
30
|
+
"n8n": {
|
|
31
|
+
"n8nNodesApiVersion": 1,
|
|
32
|
+
"strict": true,
|
|
33
|
+
"credentials": [
|
|
34
|
+
"dist/credentials/EvomiApi.credentials.js"
|
|
35
|
+
],
|
|
36
|
+
"nodes": [
|
|
37
|
+
"dist/nodes/GoogleMapsScraper/GoogleMapsScraper.node.js"
|
|
38
|
+
]
|
|
39
|
+
},
|
|
40
|
+
"dependencies": {
|
|
41
|
+
"axios": "^1.6.0"
|
|
42
|
+
},
|
|
43
|
+
"devDependencies": {
|
|
44
|
+
"@n8n/node-cli": "*",
|
|
45
|
+
"@types/node": "^18.0.0",
|
|
46
|
+
"eslint": "9.32.0",
|
|
47
|
+
"prettier": "3.6.2",
|
|
48
|
+
"release-it": "^19.0.4",
|
|
49
|
+
"typescript": "5.9.2"
|
|
50
|
+
},
|
|
51
|
+
"peerDependencies": {
|
|
52
|
+
"n8n-workflow": "*"
|
|
53
|
+
}
|
|
54
|
+
}
|