atlas-obscura-api 4.3.0 → 5.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/index.js +26 -31
- package/index.js.map +1 -1
- package/package.json +5 -6
- package/index.d.mts +0 -108
- package/index.mjs +0 -141
- package/index.mjs.map +0 -1
package/index.js
CHANGED
|
@@ -1,36 +1,32 @@
|
|
|
1
|
-
|
|
1
|
+
import { parse } from "node-html-parser";
|
|
2
|
+
import { gotScraping } from "got-scraping";
|
|
2
3
|
|
|
3
4
|
//#region src/fetchers/index.ts
|
|
4
|
-
const USER_AGENTS = [
|
|
5
|
-
"Dalvik/2.1.0 (Linux; U; Android 8.1.0; C6L Build/OPM2.171019.012)",
|
|
6
|
-
"Mozilla/5.0 (Windows; Windows NT 10.0; x64) AppleWebKit/537.50 (KHTML, like Gecko) Chrome/48.0.2572.385 Safari/537.9 Edge/8.37913",
|
|
7
|
-
"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14E5239e Safari/602.1",
|
|
8
|
-
"Dalvik/2.1.0 (Linux; U; Android 11; SM-A505GT Build/RP1A.200720.012)",
|
|
9
|
-
"Mozilla/5.0 (Windows NT 10.3;; en-US) AppleWebKit/602.25 (KHTML, like Gecko) Chrome/54.0.2457.241 Safari/534.5 Edge/9.65405",
|
|
10
|
-
"Mozilla/5.0 (Windows; U; Windows NT 10.4; x64) AppleWebKit/536.41 (KHTML, like Gecko) Chrome/53.0.3535.112 Safari/600.5 Edge/14.39786",
|
|
11
|
-
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_4_0; like Mac OS X) AppleWebKit/601.9 (KHTML, like Gecko) Chrome/47.0.2876.394 Mobile Safari/600.5",
|
|
12
|
-
"Mozilla/5.0 (iPhone; CPU iPhone OS 7_6_3; like Mac OS X) AppleWebKit/602.24 (KHTML, like Gecko) Chrome/54.0.3204.297 Mobile Safari/537.0",
|
|
13
|
-
"Mozilla/5.0 (iPhone; CPU iPhone OS 10_9_6; like Mac OS X) AppleWebKit/601.4 (KHTML, like Gecko) Chrome/55.0.2808.105 Mobile Safari/600.4",
|
|
14
|
-
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_2_1; like Mac OS X) AppleWebKit/600.14 (KHTML, like Gecko) Chrome/50.0.3808.259 Mobile Safari/534.1",
|
|
15
|
-
"Mozilla/5.0 (iPhone; CPU iPhone OS 10_4_2; like Mac OS X) AppleWebKit/534.21 (KHTML, like Gecko) Chrome/47.0.3531.351 Mobile Safari/533.0"
|
|
16
|
-
];
|
|
17
|
-
const DEFAULT_HEADERS = {
|
|
18
|
-
"User-Agent": USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)],
|
|
19
|
-
"Cache-Control": "no-cache"
|
|
20
|
-
};
|
|
21
5
|
const fetchPage = async (url, headersCustom) => {
|
|
22
6
|
try {
|
|
23
|
-
|
|
7
|
+
return (await gotScraping({
|
|
8
|
+
url,
|
|
9
|
+
headerGeneratorOptions: {
|
|
10
|
+
browsers: [{
|
|
11
|
+
name: "chrome",
|
|
12
|
+
minVersion: 115
|
|
13
|
+
}, {
|
|
14
|
+
name: "firefox",
|
|
15
|
+
minVersion: 115
|
|
16
|
+
}],
|
|
17
|
+
devices: ["desktop"],
|
|
18
|
+
locales: ["en-US"],
|
|
19
|
+
operatingSystems: ["windows", "macos"]
|
|
20
|
+
},
|
|
24
21
|
headers: {
|
|
25
|
-
|
|
22
|
+
"Cache-Control": "no-cache",
|
|
26
23
|
...headersCustom
|
|
27
24
|
},
|
|
28
|
-
|
|
29
|
-
});
|
|
30
|
-
if (!response.ok) throw new Error(`atlas-obscura-api: Bad response ${response.status} for url: ${url}`);
|
|
31
|
-
return await response.text();
|
|
25
|
+
http2: true
|
|
26
|
+
})).body;
|
|
32
27
|
} catch (error) {
|
|
33
|
-
console.error(
|
|
28
|
+
if (error.response) console.error(`atlas-obscura-api: Request failed with status code ${error.response.statusCode} for url: ${url}`);
|
|
29
|
+
else console.error("fetchPage error:", error.message);
|
|
34
30
|
return "Error";
|
|
35
31
|
}
|
|
36
32
|
};
|
|
@@ -58,10 +54,10 @@ const urlUser = (name) => `https://www.atlasobscura.com/users/${name}`;
|
|
|
58
54
|
//#region src/services/location.service.ts
|
|
59
55
|
var PlacesScraper = class {
|
|
60
56
|
async placesAll(options) {
|
|
61
|
-
return parseResults(
|
|
57
|
+
return parseResults(parse(await fetchPage(URL_PLACES_ALL, options?.headers)).innerHTML.match(/AtlasObscura\.all_places = (.*?);/));
|
|
62
58
|
}
|
|
63
59
|
async search(coords, page = 1, options) {
|
|
64
|
-
return parseResults(
|
|
60
|
+
return parseResults(parse(await fetchPage(urlSearch(coords, page), options?.headers)).innerHTML.match(/AtlasObscura\.place_search = (.*?);/));
|
|
65
61
|
}
|
|
66
62
|
async placeByIdShort(id, options) {
|
|
67
63
|
const data = await fetchPage(urlPlaceId(id), options?.headers);
|
|
@@ -75,7 +71,7 @@ var PlacesScraper = class {
|
|
|
75
71
|
return place;
|
|
76
72
|
}
|
|
77
73
|
async placeFull(url, options) {
|
|
78
|
-
const html =
|
|
74
|
+
const html = parse(await fetchPage(url, options?.headers));
|
|
79
75
|
const descriptions = html.querySelectorAll(".place-body p").map((p) => p.innerHTML);
|
|
80
76
|
const sectionDirections = Array.from(html.querySelectorAll("section")).find((sec) => {
|
|
81
77
|
const h2 = sec.querySelector("h2");
|
|
@@ -102,7 +98,7 @@ var PlacesScraper = class {
|
|
|
102
98
|
};
|
|
103
99
|
}
|
|
104
100
|
async user(user, options) {
|
|
105
|
-
return parseResults(
|
|
101
|
+
return parseResults(parse(await fetchPage(urlUser(user), options?.headers)).innerHTML.match(/AtlasObscura\.user_profile = (.*?);/)).user;
|
|
106
102
|
}
|
|
107
103
|
};
|
|
108
104
|
|
|
@@ -137,6 +133,5 @@ const locationScraper = new PlacesScraper();
|
|
|
137
133
|
const atlasObscura = new AtlasObscura(locationScraper);
|
|
138
134
|
|
|
139
135
|
//#endregion
|
|
140
|
-
|
|
141
|
-
exports.atlasObscura = atlasObscura;
|
|
136
|
+
export { AtlasObscura, atlasObscura };
|
|
142
137
|
//# sourceMappingURL=index.js.map
|
package/index.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"index.js","names":["USER_AGENTS: string[]","locations: T","place: AOPlaceShort","locationService: PlacesScraper","atlasObscura: AtlasObscura"],"sources":["../src/fetchers/index.ts","../src/utils.ts","../src/vars.ts","../src/services/location.service.ts","../src/index.ts"],"sourcesContent":["const USER_AGENTS: string[] = [\n 'Dalvik/2.1.0 (Linux; U; Android 8.1.0; C6L Build/OPM2.171019.012)',\n 'Mozilla/5.0 (Windows; Windows NT 10.0; x64) AppleWebKit/537.50 (KHTML, like Gecko) Chrome/48.0.2572.385 Safari/537.9 Edge/8.37913',\n 'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14E5239e Safari/602.1',\n 'Dalvik/2.1.0 (Linux; U; Android 11; SM-A505GT Build/RP1A.200720.012)',\n 'Mozilla/5.0 (Windows NT 10.3;; en-US) AppleWebKit/602.25 (KHTML, like Gecko) Chrome/54.0.2457.241 Safari/534.5 Edge/9.65405',\n 'Mozilla/5.0 (Windows; U; Windows NT 10.4; x64) AppleWebKit/536.41 (KHTML, like Gecko) Chrome/53.0.3535.112 Safari/600.5 Edge/14.39786',\n 'Mozilla/5.0 (iPhone; CPU iPhone OS 8_4_0; like Mac OS X) AppleWebKit/601.9 (KHTML, like Gecko) Chrome/47.0.2876.394 Mobile Safari/600.5',\n 'Mozilla/5.0 (iPhone; CPU iPhone OS 7_6_3; like Mac OS X) AppleWebKit/602.24 (KHTML, like Gecko) Chrome/54.0.3204.297 Mobile Safari/537.0',\n 'Mozilla/5.0 (iPhone; CPU iPhone OS 10_9_6; like Mac OS X) AppleWebKit/601.4 (KHTML, like Gecko) Chrome/55.0.2808.105 Mobile Safari/600.4',\n 'Mozilla/5.0 (iPhone; CPU iPhone OS 8_2_1; like Mac OS X) AppleWebKit/600.14 (KHTML, like Gecko) Chrome/50.0.3808.259 Mobile Safari/534.1',\n 'Mozilla/5.0 (iPhone; CPU iPhone OS 10_4_2; like Mac OS X) AppleWebKit/534.21 (KHTML, like Gecko) Chrome/47.0.3531.351 Mobile Safari/533.0'\n];\n\nconst DEFAULT_HEADERS = {\n 'User-Agent': USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)],\n 'Cache-Control': 'no-cache',\n};\n\nexport const fetchPage = async (url: string, headersCustom?: Headers): Promise<string> => {\n try {\n const response = await fetch(url, {\n headers: { ...DEFAULT_HEADERS, ...headersCustom },\n credentials: 'omit'\n });\n\n if (!response.ok) {\n throw new Error(`atlas-obscura-api: Bad response ${response.status} for url: ${url}`);\n }\n\n return await response.text();\n } catch (error) {\n console.error('fetchPage error:', error);\n return 'Error';\n }\n};\n","export const parseResults = <T>(matches: RegExpMatchArray): T => {\n let locations: T = null;\n\n if (matches && matches[1]?.length) {\n try {\n locations = JSON.parse(matches[1]);\n } catch (error) {\n console.error('Error parsing all places');\n }\n }\n return locations;\n};\n","export const URL_PLACES_ALL =\n 'https://www.atlasobscura.com/articles/all-places-in-the-atlas-on-one-map';\n\nexport const urlSearch = (coord: { lat: number; lng: number }, page?: number): string =>\n `https://www.atlasobscura.com/search?utf8=%E2%9C%93&q=&location=&nearby=true&lat=${coord.lat\n }&lng=${coord.lng}${page ? '&page=' + page : ''}`;\n\nexport const urlPlaceId = (id: number): string =>\n `https://www.atlasobscura.com/places/${id}.json?place_only=1`;\n\nexport const urlUser = (name: string): string =>\n `https://www.atlasobscura.com/users/${name}`;\n","import { parse } from 'node-html-parser';\nimport { fetchPage } from '../fetchers/index.js';\nimport { AOUser } from '../interfaces/user.interface.js';\nimport { parseResults } from '../utils.js';\nimport { URL_PLACES_ALL, urlPlaceId, urlSearch, urlUser } from '../vars.js';\nimport { AOCoordinates, AOFetchOptions } from './../interfaces/global.js';\nimport { AOLocationResult } from './../interfaces/locations.interface.js';\nimport { AOPlaceExtend, AOPlaceShort } from './../interfaces/placeId.interface.js';\nimport { AOPlacesAll } from './../interfaces/places-all.interface.js';\n\nexport class PlacesScraper {\n public async placesAll(options?: AOFetchOptions): Promise<AOPlacesAll[]> {\n const response = await fetchPage(URL_PLACES_ALL, options?.headers);\n const html = parse(response).innerHTML;\n\n const data = html.match(/AtlasObscura\\.all_places = (.*?);/);\n\n return parseResults<AOPlacesAll[]>(data);\n }\n\n public async search(coords: AOCoordinates, page = 1, options?: AOFetchOptions): Promise<AOLocationResult> {\n const url = urlSearch(coords, page);\n const response = await fetchPage(url, options?.headers);\n const html = parse(response).innerHTML;\n\n const data = html.match(/AtlasObscura\\.place_search = (.*?);/);\n\n return parseResults<AOLocationResult>(data);\n }\n\n public async placeByIdShort(id: number, options?: AOFetchOptions): Promise<AOPlaceShort> {\n const url = urlPlaceId(id);\n const data = await fetchPage(url, options?.headers);\n\n let place: AOPlaceShort = null;\n\n try {\n place = JSON.parse(data);\n if (place && typeof place.hide_from_maps === 'string') {\n (place as any).hide_from_maps = place.hide_from_maps === 'true';\n }\n } catch (error) {\n console.error('Error parsing placeById', id);\n }\n return place;\n }\n\n public async placeFull(url: string, options?: AOFetchOptions): Promise<AOPlaceExtend> {\n const data = await fetchPage(url, options?.headers);\n const html = parse(data);\n\n const descriptions = html.querySelectorAll('.place-body p').map((p) => p.innerHTML);\n\n const sectionDirections = Array.from(html.querySelectorAll('section')).find((sec) => {\n const h2 = sec.querySelector('h2');\n return h2 && h2.textContent.trim() === 'Know Before You Go';\n });\n const directions = sectionDirections\n ? Array.from(sectionDirections.querySelectorAll('p')).map((p) => p.textContent.trim())\n : [];\n\n const tags = html.querySelectorAll('.aon-pill-badge-component').map((x) => {\n return {\n title: x.querySelector('.aon-pill-badge-text')?.innerHTML.trim(),\n link: x.attributes.href\n };\n });\n\n const slides = html.querySelectorAll('.swiper-slide');\n const images = Array.from(slides)\n .map((slide) => {\n const img = slide.querySelector('img');\n return img ? img.getAttribute('src') : null;\n })\n .filter(Boolean);\n\n const imageCover = images[0];\n\n return { description: descriptions, directions, tags, imageCover, images };\n }\n\n public async user(user: string, options?: AOFetchOptions): Promise<AOUser> {\n const url = urlUser(user);\n const response = await fetchPage(url, options?.headers);\n const html = parse(response).innerHTML;\n\n const data = html.match(/AtlasObscura\\.user_profile = (.*?);/);\n\n const usr = parseResults<{ user: AOUser }>(data);\n\n return usr.user;\n }\n}\n","import { AOCoordinates, AOFetchOptions } from './interfaces/global.js';\nimport { AOLocationResult } from './interfaces/locations.interface.js';\nimport { AOPlace, AOPlaceShort } from './interfaces/placeId.interface.js';\nimport { AOPlacesAll } from './interfaces/places-all.interface.js';\nimport { AOUser } from './interfaces/user.interface.js';\nimport { PlacesScraper } from './services/location.service.js';\n\nexport class AtlasObscura {\n constructor(private locationService: PlacesScraper) { }\n\n public async search(coords: AOCoordinates, options?: AOFetchOptions): Promise<AOLocationResult> {\n return this.locationService.search(coords, 1, options);\n }\n\n public async placesAll(options?: AOFetchOptions): Promise<AOPlacesAll[]> {\n return this.locationService.placesAll(options);\n }\n\n public async placeShort(id: number, options?: AOFetchOptions): Promise<AOPlaceShort> {\n return this.locationService.placeByIdShort(id, options);\n }\n\n public async placeFull(id: number, options?: AOFetchOptions): Promise<AOPlace> {\n const place = await this.locationService.placeByIdShort(id, options);\n const placeRich = await this.locationService.placeFull(place.url, options);\n return { ...place, ...placeRich };\n }\n\n public async user(user: string, options?: AOFetchOptions): Promise<AOUser> {\n return this.locationService.user(user, options);\n }\n}\n\nconst locationScraper = new PlacesScraper();\nexport const atlasObscura: AtlasObscura = new AtlasObscura(locationScraper);\nexport type * from './interfaces/index.js';\n"],"mappings":";;;AAAA,MAAMA,cAAwB;CAC5B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACD;AAED,MAAM,kBAAkB;CACtB,cAAc,YAAY,KAAK,MAAM,KAAK,QAAQ,GAAG,YAAY,OAAO;CACxE,iBAAiB;CAClB;AAED,MAAa,YAAY,OAAO,KAAa,kBAA6C;AACxF,KAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK;GAChC,SAAS;IAAE,GAAG;IAAiB,GAAG;IAAe;GACjD,aAAa;GACd,CAAC;AAEF,MAAI,CAAC,SAAS,GACZ,OAAM,IAAI,MAAM,mCAAmC,SAAS,OAAO,YAAY,MAAM;AAGvF,SAAO,MAAM,SAAS,MAAM;UACrB,OAAO;AACd,UAAQ,MAAM,oBAAoB,MAAM;AACxC,SAAO;;;;;;ACjCX,MAAa,gBAAmB,YAAiC;CAC/D,IAAIC,YAAe;AAEnB,KAAI,WAAW,QAAQ,IAAI,OACzB,KAAI;AACF,cAAY,KAAK,MAAM,QAAQ,GAAG;UAC3B,OAAO;AACd,UAAQ,MAAM,2BAA2B;;AAG7C,QAAO;;;;;ACVT,MAAa,iBACX;AAEF,MAAa,aAAa,OAAqC,SAC7D,mFAAmF,MAAM,IACxF,OAAO,MAAM,MAAM,OAAO,WAAW,OAAO;AAE/C,MAAa,cAAc,OACzB,uCAAuC,GAAG;AAE5C,MAAa,WAAW,SACtB,sCAAsC;;;;ACDxC,IAAa,gBAAb,MAA2B;CACzB,MAAa,UAAU,SAAkD;AAMvE,SAAO,yCALU,MAAM,UAAU,gBAAgB,SAAS,QAAQ,CACtC,CAAC,UAEX,MAAM,oCAAoC,CAEpB;;CAG1C,MAAa,OAAO,QAAuB,OAAO,GAAG,SAAqD;AAOxG,SAAO,yCALU,MAAM,UADX,UAAU,QAAQ,KAAK,EACG,SAAS,QAAQ,CAC3B,CAAC,UAEX,MAAM,sCAAsC,CAEnB;;CAG7C,MAAa,eAAe,IAAY,SAAiD;EAEvF,MAAM,OAAO,MAAM,UADP,WAAW,GAAG,EACQ,SAAS,QAAQ;EAEnD,IAAIC,QAAsB;AAE1B,MAAI;AACF,WAAQ,KAAK,MAAM,KAAK;AACxB,OAAI,SAAS,OAAO,MAAM,mBAAmB,SAC3C,CAAC,MAAc,iBAAiB,MAAM,mBAAmB;WAEpD,OAAO;AACd,WAAQ,MAAM,2BAA2B,GAAG;;AAE9C,SAAO;;CAGT,MAAa,UAAU,KAAa,SAAkD;EAEpF,MAAM,mCADO,MAAM,UAAU,KAAK,SAAS,QAAQ,CAC3B;EAExB,MAAM,eAAe,KAAK,iBAAiB,gBAAgB,CAAC,KAAK,MAAM,EAAE,UAAU;EAEnF,MAAM,oBAAoB,MAAM,KAAK,KAAK,iBAAiB,UAAU,CAAC,CAAC,MAAM,QAAQ;GACnF,MAAM,KAAK,IAAI,cAAc,KAAK;AAClC,UAAO,MAAM,GAAG,YAAY,MAAM,KAAK;IACvC;EACF,MAAM,aAAa,oBACf,MAAM,KAAK,kBAAkB,iBAAiB,IAAI,CAAC,CAAC,KAAK,MAAM,EAAE,YAAY,MAAM,CAAC,GACpF,EAAE;EAEN,MAAM,OAAO,KAAK,iBAAiB,4BAA4B,CAAC,KAAK,MAAM;AACzE,UAAO;IACL,OAAO,EAAE,cAAc,uBAAuB,EAAE,UAAU,MAAM;IAChE,MAAM,EAAE,WAAW;IACpB;IACD;EAEF,MAAM,SAAS,KAAK,iBAAiB,gBAAgB;EACrD,MAAM,SAAS,MAAM,KAAK,OAAO,CAC9B,KAAK,UAAU;GACd,MAAM,MAAM,MAAM,cAAc,MAAM;AACtC,UAAO,MAAM,IAAI,aAAa,MAAM,GAAG;IACvC,CACD,OAAO,QAAQ;AAIlB,SAAO;GAAE,aAAa;GAAc;GAAY;GAAM,YAFnC,OAAO;GAEwC;GAAQ;;CAG5E,MAAa,KAAK,MAAc,SAA2C;AASzE,SAFY,yCALK,MAAM,UADX,QAAQ,KAAK,EACa,SAAS,QAAQ,CAC3B,CAAC,UAEX,MAAM,sCAAsC,CAEd,CAErC;;;;;;ACnFf,IAAa,eAAb,MAA0B;CACxB,YAAY,AAAQC,iBAAgC;EAAhC;;CAEpB,MAAa,OAAO,QAAuB,SAAqD;AAC9F,SAAO,KAAK,gBAAgB,OAAO,QAAQ,GAAG,QAAQ;;CAGxD,MAAa,UAAU,SAAkD;AACvE,SAAO,KAAK,gBAAgB,UAAU,QAAQ;;CAGhD,MAAa,WAAW,IAAY,SAAiD;AACnF,SAAO,KAAK,gBAAgB,eAAe,IAAI,QAAQ;;CAGzD,MAAa,UAAU,IAAY,SAA4C;EAC7E,MAAM,QAAQ,MAAM,KAAK,gBAAgB,eAAe,IAAI,QAAQ;EACpE,MAAM,YAAY,MAAM,KAAK,gBAAgB,UAAU,MAAM,KAAK,QAAQ;AAC1E,SAAO;GAAE,GAAG;GAAO,GAAG;GAAW;;CAGnC,MAAa,KAAK,MAAc,SAA2C;AACzE,SAAO,KAAK,gBAAgB,KAAK,MAAM,QAAQ;;;AAInD,MAAM,kBAAkB,IAAI,eAAe;AAC3C,MAAaC,eAA6B,IAAI,aAAa,gBAAgB"}
|
|
1
|
+
{"version":3,"file":"index.js","names":["error: any","locations: T","place: AOPlaceShort","locationService: PlacesScraper","atlasObscura: AtlasObscura"],"sources":["../src/fetchers/index.ts","../src/utils.ts","../src/vars.ts","../src/services/location.service.ts","../src/index.ts"],"sourcesContent":["import { gotScraping } from 'got-scraping';\n\nexport const fetchPage = async (url: string, headersCustom?: Record<string, string>): Promise<string> => {\n try {\n const response = await gotScraping({\n url,\n // Configuration to mimic a modern desktop browser (Chrome/Firefox)\n headerGeneratorOptions: {\n browsers: [\n { name: 'chrome', minVersion: 115 },\n { name: 'firefox', minVersion: 115 }\n ],\n devices: ['desktop'],\n locales: ['en-US'],\n operatingSystems: ['windows', 'macos'],\n },\n headers: {\n 'Cache-Control': 'no-cache',\n ...headersCustom,\n },\n // Optional: HTTP2 is often faster and less suspicious for modern sites,\n // but sometimes causes stability issues. Try true if false doesn't work.\n http2: true,\n });\n\n return response.body;\n\n } catch (error: any) {\n // Better error logging to debug 403s\n if (error.response) {\n console.error(`atlas-obscura-api: Request failed with status code ${error.response.statusCode} for url: ${url}`);\n } else {\n console.error('fetchPage error:', error.message);\n }\n\n return 'Error';\n }\n};","export const parseResults = <T>(matches: RegExpMatchArray): T => {\n let locations: T = null;\n\n if (matches && matches[1]?.length) {\n try {\n locations = JSON.parse(matches[1]);\n } catch (error) {\n console.error('Error parsing all places');\n }\n }\n return locations;\n};\n","export const URL_PLACES_ALL =\n 'https://www.atlasobscura.com/articles/all-places-in-the-atlas-on-one-map';\n\nexport const urlSearch = (coord: { lat: number; lng: number }, page?: number): string =>\n `https://www.atlasobscura.com/search?utf8=%E2%9C%93&q=&location=&nearby=true&lat=${coord.lat\n }&lng=${coord.lng}${page ? '&page=' + page : ''}`;\n\nexport const urlPlaceId = (id: number): string =>\n `https://www.atlasobscura.com/places/${id}.json?place_only=1`;\n\nexport const urlUser = (name: string): string =>\n `https://www.atlasobscura.com/users/${name}`;\n","import { parse } from 'node-html-parser';\nimport { fetchPage } from '../fetchers/index.js';\nimport { AOUser } from '../interfaces/user.interface.js';\nimport { parseResults } from '../utils.js';\nimport { URL_PLACES_ALL, urlPlaceId, urlSearch, urlUser } from '../vars.js';\nimport { AOCoordinates, AOFetchOptions } from './../interfaces/global.js';\nimport { AOLocationResult } from './../interfaces/locations.interface.js';\nimport { AOPlaceExtend, AOPlaceShort } from './../interfaces/placeId.interface.js';\nimport { AOPlacesAll } from './../interfaces/places-all.interface.js';\n\nexport class PlacesScraper {\n public async placesAll(options?: AOFetchOptions): Promise<AOPlacesAll[]> {\n const response = await fetchPage(URL_PLACES_ALL, options?.headers);\n const html = parse(response).innerHTML;\n\n const data = html.match(/AtlasObscura\\.all_places = (.*?);/);\n\n return parseResults<AOPlacesAll[]>(data);\n }\n\n public async search(coords: AOCoordinates, page = 1, options?: AOFetchOptions): Promise<AOLocationResult> {\n const url = urlSearch(coords, page);\n const response = await fetchPage(url, options?.headers);\n const html = parse(response).innerHTML;\n\n const data = html.match(/AtlasObscura\\.place_search = (.*?);/);\n\n return parseResults<AOLocationResult>(data);\n }\n\n public async placeByIdShort(id: number, options?: AOFetchOptions): Promise<AOPlaceShort> {\n const url = urlPlaceId(id);\n const data = await fetchPage(url, options?.headers);\n\n let place: AOPlaceShort = null;\n\n try {\n place = JSON.parse(data);\n if (place && typeof place.hide_from_maps === 'string') {\n (place as any).hide_from_maps = place.hide_from_maps === 'true';\n }\n } catch (error) {\n console.error('Error parsing placeById', id);\n }\n return place;\n }\n\n public async placeFull(url: string, options?: AOFetchOptions): Promise<AOPlaceExtend> {\n const data = await fetchPage(url, options?.headers);\n const html = parse(data);\n\n const descriptions = html.querySelectorAll('.place-body p').map((p) => p.innerHTML);\n\n const sectionDirections = Array.from(html.querySelectorAll('section')).find((sec) => {\n const h2 = sec.querySelector('h2');\n return h2 && h2.textContent.trim() === 'Know Before You Go';\n });\n const directions = sectionDirections\n ? Array.from(sectionDirections.querySelectorAll('p')).map((p) => p.textContent.trim())\n : [];\n\n const tags = html.querySelectorAll('.aon-pill-badge-component').map((x) => {\n return {\n title: x.querySelector('.aon-pill-badge-text')?.innerHTML.trim(),\n link: x.attributes.href\n };\n });\n\n const slides = html.querySelectorAll('.swiper-slide');\n const images = Array.from(slides)\n .map((slide) => {\n const img = slide.querySelector('img');\n return img ? img.getAttribute('src') : null;\n })\n .filter(Boolean);\n\n const imageCover = images[0];\n\n return { description: descriptions, directions, tags, imageCover, images };\n }\n\n public async user(user: string, options?: AOFetchOptions): Promise<AOUser> {\n const url = urlUser(user);\n const response = await fetchPage(url, options?.headers);\n const html = parse(response).innerHTML;\n\n const data = html.match(/AtlasObscura\\.user_profile = (.*?);/);\n\n const usr = parseResults<{ user: AOUser }>(data);\n\n return usr.user;\n }\n}\n","import { AOCoordinates, AOFetchOptions } from './interfaces/global.js';\nimport { AOLocationResult } from './interfaces/locations.interface.js';\nimport { AOPlace, AOPlaceShort } from './interfaces/placeId.interface.js';\nimport { AOPlacesAll } from './interfaces/places-all.interface.js';\nimport { AOUser } from './interfaces/user.interface.js';\nimport { PlacesScraper } from './services/location.service.js';\n\nexport class AtlasObscura {\n constructor(private locationService: PlacesScraper) { }\n\n public async search(coords: AOCoordinates, options?: AOFetchOptions): Promise<AOLocationResult> {\n return this.locationService.search(coords, 1, options);\n }\n\n public async placesAll(options?: AOFetchOptions): Promise<AOPlacesAll[]> {\n return this.locationService.placesAll(options);\n }\n\n public async placeShort(id: number, options?: AOFetchOptions): Promise<AOPlaceShort> {\n return this.locationService.placeByIdShort(id, options);\n }\n\n public async placeFull(id: number, options?: AOFetchOptions): Promise<AOPlace> {\n const place = await this.locationService.placeByIdShort(id, options);\n const placeRich = await this.locationService.placeFull(place.url, options);\n return { ...place, ...placeRich };\n }\n\n public async user(user: string, options?: AOFetchOptions): Promise<AOUser> {\n return this.locationService.user(user, options);\n }\n}\n\nconst locationScraper = new PlacesScraper();\nexport const atlasObscura: AtlasObscura = new AtlasObscura(locationScraper);\nexport type * from './interfaces/index.js';\n"],"mappings":";;;;AAEA,MAAa,YAAY,OAAO,KAAa,kBAA4D;AACvG,KAAI;AAsBF,UArBiB,MAAM,YAAY;GACjC;GAEA,wBAAwB;IACtB,UAAU,CACR;KAAE,MAAM;KAAU,YAAY;KAAK,EACnC;KAAE,MAAM;KAAW,YAAY;KAAK,CACrC;IACD,SAAS,CAAC,UAAU;IACpB,SAAS,CAAC,QAAQ;IAClB,kBAAkB,CAAC,WAAW,QAAQ;IACvC;GACD,SAAS;IACP,iBAAiB;IACjB,GAAG;IACJ;GAGD,OAAO;GACR,CAAC,EAEc;UAETA,OAAY;AAEnB,MAAI,MAAM,SACR,SAAQ,MAAM,sDAAsD,MAAM,SAAS,WAAW,YAAY,MAAM;MAEhH,SAAQ,MAAM,oBAAoB,MAAM,QAAQ;AAGlD,SAAO;;;;;;ACnCX,MAAa,gBAAmB,YAAiC;CAC/D,IAAIC,YAAe;AAEnB,KAAI,WAAW,QAAQ,IAAI,OACzB,KAAI;AACF,cAAY,KAAK,MAAM,QAAQ,GAAG;UAC3B,OAAO;AACd,UAAQ,MAAM,2BAA2B;;AAG7C,QAAO;;;;;ACVT,MAAa,iBACX;AAEF,MAAa,aAAa,OAAqC,SAC7D,mFAAmF,MAAM,IACxF,OAAO,MAAM,MAAM,OAAO,WAAW,OAAO;AAE/C,MAAa,cAAc,OACzB,uCAAuC,GAAG;AAE5C,MAAa,WAAW,SACtB,sCAAsC;;;;ACDxC,IAAa,gBAAb,MAA2B;CACzB,MAAa,UAAU,SAAkD;AAMvE,SAAO,aAJM,MADI,MAAM,UAAU,gBAAgB,SAAS,QAAQ,CACtC,CAAC,UAEX,MAAM,oCAAoC,CAEpB;;CAG1C,MAAa,OAAO,QAAuB,OAAO,GAAG,SAAqD;AAOxG,SAAO,aAJM,MADI,MAAM,UADX,UAAU,QAAQ,KAAK,EACG,SAAS,QAAQ,CAC3B,CAAC,UAEX,MAAM,sCAAsC,CAEnB;;CAG7C,MAAa,eAAe,IAAY,SAAiD;EAEvF,MAAM,OAAO,MAAM,UADP,WAAW,GAAG,EACQ,SAAS,QAAQ;EAEnD,IAAIC,QAAsB;AAE1B,MAAI;AACF,WAAQ,KAAK,MAAM,KAAK;AACxB,OAAI,SAAS,OAAO,MAAM,mBAAmB,SAC3C,CAAC,MAAc,iBAAiB,MAAM,mBAAmB;WAEpD,OAAO;AACd,WAAQ,MAAM,2BAA2B,GAAG;;AAE9C,SAAO;;CAGT,MAAa,UAAU,KAAa,SAAkD;EAEpF,MAAM,OAAO,MADA,MAAM,UAAU,KAAK,SAAS,QAAQ,CAC3B;EAExB,MAAM,eAAe,KAAK,iBAAiB,gBAAgB,CAAC,KAAK,MAAM,EAAE,UAAU;EAEnF,MAAM,oBAAoB,MAAM,KAAK,KAAK,iBAAiB,UAAU,CAAC,CAAC,MAAM,QAAQ;GACnF,MAAM,KAAK,IAAI,cAAc,KAAK;AAClC,UAAO,MAAM,GAAG,YAAY,MAAM,KAAK;IACvC;EACF,MAAM,aAAa,oBACf,MAAM,KAAK,kBAAkB,iBAAiB,IAAI,CAAC,CAAC,KAAK,MAAM,EAAE,YAAY,MAAM,CAAC,GACpF,EAAE;EAEN,MAAM,OAAO,KAAK,iBAAiB,4BAA4B,CAAC,KAAK,MAAM;AACzE,UAAO;IACL,OAAO,EAAE,cAAc,uBAAuB,EAAE,UAAU,MAAM;IAChE,MAAM,EAAE,WAAW;IACpB;IACD;EAEF,MAAM,SAAS,KAAK,iBAAiB,gBAAgB;EACrD,MAAM,SAAS,MAAM,KAAK,OAAO,CAC9B,KAAK,UAAU;GACd,MAAM,MAAM,MAAM,cAAc,MAAM;AACtC,UAAO,MAAM,IAAI,aAAa,MAAM,GAAG;IACvC,CACD,OAAO,QAAQ;AAIlB,SAAO;GAAE,aAAa;GAAc;GAAY;GAAM,YAFnC,OAAO;GAEwC;GAAQ;;CAG5E,MAAa,KAAK,MAAc,SAA2C;AASzE,SAFY,aAJC,MADI,MAAM,UADX,QAAQ,KAAK,EACa,SAAS,QAAQ,CAC3B,CAAC,UAEX,MAAM,sCAAsC,CAEd,CAErC;;;;;;ACnFf,IAAa,eAAb,MAA0B;CACxB,YAAY,AAAQC,iBAAgC;EAAhC;;CAEpB,MAAa,OAAO,QAAuB,SAAqD;AAC9F,SAAO,KAAK,gBAAgB,OAAO,QAAQ,GAAG,QAAQ;;CAGxD,MAAa,UAAU,SAAkD;AACvE,SAAO,KAAK,gBAAgB,UAAU,QAAQ;;CAGhD,MAAa,WAAW,IAAY,SAAiD;AACnF,SAAO,KAAK,gBAAgB,eAAe,IAAI,QAAQ;;CAGzD,MAAa,UAAU,IAAY,SAA4C;EAC7E,MAAM,QAAQ,MAAM,KAAK,gBAAgB,eAAe,IAAI,QAAQ;EACpE,MAAM,YAAY,MAAM,KAAK,gBAAgB,UAAU,MAAM,KAAK,QAAQ;AAC1E,SAAO;GAAE,GAAG;GAAO,GAAG;GAAW;;CAGnC,MAAa,KAAK,MAAc,SAA2C;AACzE,SAAO,KAAK,gBAAgB,KAAK,MAAM,QAAQ;;;AAInD,MAAM,kBAAkB,IAAI,eAAe;AAC3C,MAAaC,eAA6B,IAAI,aAAa,gBAAgB"}
|
package/package.json
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "atlas-obscura-api",
|
|
3
|
-
"version": "
|
|
3
|
+
"version": "5.0.0",
|
|
4
4
|
"description": "Atlas Obscura API in JavaScript. NPM library for scrapping atlasobscura.com",
|
|
5
5
|
"author": "BART! <bart@bartweb.cz>",
|
|
6
|
+
"type": "module",
|
|
6
7
|
"dependencies": {
|
|
8
|
+
"got-scraping": "^4.1.2",
|
|
7
9
|
"node-html-parser": "^7.0.1"
|
|
8
10
|
},
|
|
9
11
|
"repository": {
|
|
@@ -31,13 +33,10 @@
|
|
|
31
33
|
},
|
|
32
34
|
"license": "MIT",
|
|
33
35
|
"main": "./index.js",
|
|
34
|
-
"module": "./index.
|
|
36
|
+
"module": "./index.js",
|
|
35
37
|
"types": "./index.d.ts",
|
|
36
38
|
"exports": {
|
|
37
|
-
".":
|
|
38
|
-
"require": "./index.js",
|
|
39
|
-
"import": "./index.mjs"
|
|
40
|
-
},
|
|
39
|
+
".": "./index.js",
|
|
41
40
|
"./package.json": "./package.json"
|
|
42
41
|
}
|
|
43
42
|
}
|
package/index.d.mts
DELETED
|
@@ -1,108 +0,0 @@
|
|
|
1
|
-
//#region src/interfaces/global.d.ts
|
|
2
|
-
interface AOCoordinates {
|
|
3
|
-
lat: number;
|
|
4
|
-
lng: number;
|
|
5
|
-
}
|
|
6
|
-
type AOFetchOptions = {
|
|
7
|
-
headers?: Headers;
|
|
8
|
-
};
|
|
9
|
-
//#endregion
|
|
10
|
-
//#region src/interfaces/locations.interface.d.ts
|
|
11
|
-
interface AOLocationResult {
|
|
12
|
-
total: AOTotal;
|
|
13
|
-
per_page: number;
|
|
14
|
-
current_page: number;
|
|
15
|
-
location: string;
|
|
16
|
-
q: string;
|
|
17
|
-
coordinates: AOCoordinates;
|
|
18
|
-
results: AOLocation[];
|
|
19
|
-
}
|
|
20
|
-
interface AOLocation {
|
|
21
|
-
title: string;
|
|
22
|
-
subtitle: string;
|
|
23
|
-
location: string;
|
|
24
|
-
thumbnail_url: string;
|
|
25
|
-
url: string;
|
|
26
|
-
id: number;
|
|
27
|
-
hide_from_maps?: string;
|
|
28
|
-
coordinates: AOCoordinates;
|
|
29
|
-
distance_from_query: string;
|
|
30
|
-
}
|
|
31
|
-
interface AOTotal {
|
|
32
|
-
value: number;
|
|
33
|
-
relation: string;
|
|
34
|
-
}
|
|
35
|
-
//#endregion
|
|
36
|
-
//#region src/interfaces/placeId.interface.d.ts
|
|
37
|
-
interface AOPlaceShort {
|
|
38
|
-
id: number;
|
|
39
|
-
title: string;
|
|
40
|
-
subtitle: string;
|
|
41
|
-
city: string;
|
|
42
|
-
country: string;
|
|
43
|
-
location: string;
|
|
44
|
-
url: string;
|
|
45
|
-
hide_from_maps?: string;
|
|
46
|
-
physical_status: string;
|
|
47
|
-
thumbnail_url: string;
|
|
48
|
-
thumbnail_url_3x2: string;
|
|
49
|
-
coordinates: AOCoordinates;
|
|
50
|
-
nearby_places: AOPlaceShort[];
|
|
51
|
-
nearby_foods: AOPlaceShort[];
|
|
52
|
-
}
|
|
53
|
-
interface AOPlaceExtend {
|
|
54
|
-
description?: string[];
|
|
55
|
-
tags?: AOTag[];
|
|
56
|
-
directions?: string[];
|
|
57
|
-
imageCover?: string;
|
|
58
|
-
images?: string[];
|
|
59
|
-
}
|
|
60
|
-
interface AOTag {
|
|
61
|
-
title: string;
|
|
62
|
-
link: string;
|
|
63
|
-
}
|
|
64
|
-
interface AOPlace extends AOPlaceShort, AOPlaceExtend {}
|
|
65
|
-
//#endregion
|
|
66
|
-
//#region src/interfaces/places-all.interface.d.ts
|
|
67
|
-
interface AOPlacesAll extends AOCoordinates {
|
|
68
|
-
id: number;
|
|
69
|
-
}
|
|
70
|
-
//#endregion
|
|
71
|
-
//#region src/interfaces/user.interface.d.ts
|
|
72
|
-
interface AOUser {
|
|
73
|
-
id: number;
|
|
74
|
-
places: AOUserPlace[];
|
|
75
|
-
}
|
|
76
|
-
interface AOUserPlace {
|
|
77
|
-
id: number;
|
|
78
|
-
title: string;
|
|
79
|
-
subtitle: string;
|
|
80
|
-
url: string;
|
|
81
|
-
location: string;
|
|
82
|
-
thumbnail_url: string;
|
|
83
|
-
coordinates: AOCoordinates;
|
|
84
|
-
}
|
|
85
|
-
//#endregion
|
|
86
|
-
//#region src/services/location.service.d.ts
|
|
87
|
-
declare class PlacesScraper {
|
|
88
|
-
placesAll(options?: AOFetchOptions): Promise<AOPlacesAll[]>;
|
|
89
|
-
search(coords: AOCoordinates, page?: number, options?: AOFetchOptions): Promise<AOLocationResult>;
|
|
90
|
-
placeByIdShort(id: number, options?: AOFetchOptions): Promise<AOPlaceShort>;
|
|
91
|
-
placeFull(url: string, options?: AOFetchOptions): Promise<AOPlaceExtend>;
|
|
92
|
-
user(user: string, options?: AOFetchOptions): Promise<AOUser>;
|
|
93
|
-
}
|
|
94
|
-
//#endregion
|
|
95
|
-
//#region src/index.d.ts
|
|
96
|
-
declare class AtlasObscura {
|
|
97
|
-
private locationService;
|
|
98
|
-
constructor(locationService: PlacesScraper);
|
|
99
|
-
search(coords: AOCoordinates, options?: AOFetchOptions): Promise<AOLocationResult>;
|
|
100
|
-
placesAll(options?: AOFetchOptions): Promise<AOPlacesAll[]>;
|
|
101
|
-
placeShort(id: number, options?: AOFetchOptions): Promise<AOPlaceShort>;
|
|
102
|
-
placeFull(id: number, options?: AOFetchOptions): Promise<AOPlace>;
|
|
103
|
-
user(user: string, options?: AOFetchOptions): Promise<AOUser>;
|
|
104
|
-
}
|
|
105
|
-
declare const atlasObscura: AtlasObscura;
|
|
106
|
-
//#endregion
|
|
107
|
-
export { AOCoordinates, AOFetchOptions, AOLocation, AOLocationResult, AOPlace, AOPlaceExtend, AOPlaceShort, AOPlacesAll, AOTag, AOTotal, AtlasObscura, atlasObscura };
|
|
108
|
-
//# sourceMappingURL=index.d.mts.map
|
package/index.mjs
DELETED
|
@@ -1,141 +0,0 @@
|
|
|
1
|
-
import { parse } from "node-html-parser";
|
|
2
|
-
|
|
3
|
-
//#region src/fetchers/index.ts
|
|
4
|
-
const USER_AGENTS = [
|
|
5
|
-
"Dalvik/2.1.0 (Linux; U; Android 8.1.0; C6L Build/OPM2.171019.012)",
|
|
6
|
-
"Mozilla/5.0 (Windows; Windows NT 10.0; x64) AppleWebKit/537.50 (KHTML, like Gecko) Chrome/48.0.2572.385 Safari/537.9 Edge/8.37913",
|
|
7
|
-
"Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14E5239e Safari/602.1",
|
|
8
|
-
"Dalvik/2.1.0 (Linux; U; Android 11; SM-A505GT Build/RP1A.200720.012)",
|
|
9
|
-
"Mozilla/5.0 (Windows NT 10.3;; en-US) AppleWebKit/602.25 (KHTML, like Gecko) Chrome/54.0.2457.241 Safari/534.5 Edge/9.65405",
|
|
10
|
-
"Mozilla/5.0 (Windows; U; Windows NT 10.4; x64) AppleWebKit/536.41 (KHTML, like Gecko) Chrome/53.0.3535.112 Safari/600.5 Edge/14.39786",
|
|
11
|
-
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_4_0; like Mac OS X) AppleWebKit/601.9 (KHTML, like Gecko) Chrome/47.0.2876.394 Mobile Safari/600.5",
|
|
12
|
-
"Mozilla/5.0 (iPhone; CPU iPhone OS 7_6_3; like Mac OS X) AppleWebKit/602.24 (KHTML, like Gecko) Chrome/54.0.3204.297 Mobile Safari/537.0",
|
|
13
|
-
"Mozilla/5.0 (iPhone; CPU iPhone OS 10_9_6; like Mac OS X) AppleWebKit/601.4 (KHTML, like Gecko) Chrome/55.0.2808.105 Mobile Safari/600.4",
|
|
14
|
-
"Mozilla/5.0 (iPhone; CPU iPhone OS 8_2_1; like Mac OS X) AppleWebKit/600.14 (KHTML, like Gecko) Chrome/50.0.3808.259 Mobile Safari/534.1",
|
|
15
|
-
"Mozilla/5.0 (iPhone; CPU iPhone OS 10_4_2; like Mac OS X) AppleWebKit/534.21 (KHTML, like Gecko) Chrome/47.0.3531.351 Mobile Safari/533.0"
|
|
16
|
-
];
|
|
17
|
-
const DEFAULT_HEADERS = {
|
|
18
|
-
"User-Agent": USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)],
|
|
19
|
-
"Cache-Control": "no-cache"
|
|
20
|
-
};
|
|
21
|
-
const fetchPage = async (url, headersCustom) => {
|
|
22
|
-
try {
|
|
23
|
-
const response = await fetch(url, {
|
|
24
|
-
headers: {
|
|
25
|
-
...DEFAULT_HEADERS,
|
|
26
|
-
...headersCustom
|
|
27
|
-
},
|
|
28
|
-
credentials: "omit"
|
|
29
|
-
});
|
|
30
|
-
if (!response.ok) throw new Error(`atlas-obscura-api: Bad response ${response.status} for url: ${url}`);
|
|
31
|
-
return await response.text();
|
|
32
|
-
} catch (error) {
|
|
33
|
-
console.error("fetchPage error:", error);
|
|
34
|
-
return "Error";
|
|
35
|
-
}
|
|
36
|
-
};
|
|
37
|
-
|
|
38
|
-
//#endregion
|
|
39
|
-
//#region src/utils.ts
|
|
40
|
-
const parseResults = (matches) => {
|
|
41
|
-
let locations = null;
|
|
42
|
-
if (matches && matches[1]?.length) try {
|
|
43
|
-
locations = JSON.parse(matches[1]);
|
|
44
|
-
} catch (error) {
|
|
45
|
-
console.error("Error parsing all places");
|
|
46
|
-
}
|
|
47
|
-
return locations;
|
|
48
|
-
};
|
|
49
|
-
|
|
50
|
-
//#endregion
|
|
51
|
-
//#region src/vars.ts
|
|
52
|
-
const URL_PLACES_ALL = "https://www.atlasobscura.com/articles/all-places-in-the-atlas-on-one-map";
|
|
53
|
-
const urlSearch = (coord, page) => `https://www.atlasobscura.com/search?utf8=%E2%9C%93&q=&location=&nearby=true&lat=${coord.lat}&lng=${coord.lng}${page ? "&page=" + page : ""}`;
|
|
54
|
-
const urlPlaceId = (id) => `https://www.atlasobscura.com/places/${id}.json?place_only=1`;
|
|
55
|
-
const urlUser = (name) => `https://www.atlasobscura.com/users/${name}`;
|
|
56
|
-
|
|
57
|
-
//#endregion
|
|
58
|
-
//#region src/services/location.service.ts
|
|
59
|
-
var PlacesScraper = class {
|
|
60
|
-
async placesAll(options) {
|
|
61
|
-
return parseResults(parse(await fetchPage(URL_PLACES_ALL, options?.headers)).innerHTML.match(/AtlasObscura\.all_places = (.*?);/));
|
|
62
|
-
}
|
|
63
|
-
async search(coords, page = 1, options) {
|
|
64
|
-
return parseResults(parse(await fetchPage(urlSearch(coords, page), options?.headers)).innerHTML.match(/AtlasObscura\.place_search = (.*?);/));
|
|
65
|
-
}
|
|
66
|
-
async placeByIdShort(id, options) {
|
|
67
|
-
const data = await fetchPage(urlPlaceId(id), options?.headers);
|
|
68
|
-
let place = null;
|
|
69
|
-
try {
|
|
70
|
-
place = JSON.parse(data);
|
|
71
|
-
if (place && typeof place.hide_from_maps === "string") place.hide_from_maps = place.hide_from_maps === "true";
|
|
72
|
-
} catch (error) {
|
|
73
|
-
console.error("Error parsing placeById", id);
|
|
74
|
-
}
|
|
75
|
-
return place;
|
|
76
|
-
}
|
|
77
|
-
async placeFull(url, options) {
|
|
78
|
-
const html = parse(await fetchPage(url, options?.headers));
|
|
79
|
-
const descriptions = html.querySelectorAll(".place-body p").map((p) => p.innerHTML);
|
|
80
|
-
const sectionDirections = Array.from(html.querySelectorAll("section")).find((sec) => {
|
|
81
|
-
const h2 = sec.querySelector("h2");
|
|
82
|
-
return h2 && h2.textContent.trim() === "Know Before You Go";
|
|
83
|
-
});
|
|
84
|
-
const directions = sectionDirections ? Array.from(sectionDirections.querySelectorAll("p")).map((p) => p.textContent.trim()) : [];
|
|
85
|
-
const tags = html.querySelectorAll(".aon-pill-badge-component").map((x) => {
|
|
86
|
-
return {
|
|
87
|
-
title: x.querySelector(".aon-pill-badge-text")?.innerHTML.trim(),
|
|
88
|
-
link: x.attributes.href
|
|
89
|
-
};
|
|
90
|
-
});
|
|
91
|
-
const slides = html.querySelectorAll(".swiper-slide");
|
|
92
|
-
const images = Array.from(slides).map((slide) => {
|
|
93
|
-
const img = slide.querySelector("img");
|
|
94
|
-
return img ? img.getAttribute("src") : null;
|
|
95
|
-
}).filter(Boolean);
|
|
96
|
-
return {
|
|
97
|
-
description: descriptions,
|
|
98
|
-
directions,
|
|
99
|
-
tags,
|
|
100
|
-
imageCover: images[0],
|
|
101
|
-
images
|
|
102
|
-
};
|
|
103
|
-
}
|
|
104
|
-
async user(user, options) {
|
|
105
|
-
return parseResults(parse(await fetchPage(urlUser(user), options?.headers)).innerHTML.match(/AtlasObscura\.user_profile = (.*?);/)).user;
|
|
106
|
-
}
|
|
107
|
-
};
|
|
108
|
-
|
|
109
|
-
//#endregion
|
|
110
|
-
//#region src/index.ts
|
|
111
|
-
var AtlasObscura = class {
|
|
112
|
-
constructor(locationService) {
|
|
113
|
-
this.locationService = locationService;
|
|
114
|
-
}
|
|
115
|
-
async search(coords, options) {
|
|
116
|
-
return this.locationService.search(coords, 1, options);
|
|
117
|
-
}
|
|
118
|
-
async placesAll(options) {
|
|
119
|
-
return this.locationService.placesAll(options);
|
|
120
|
-
}
|
|
121
|
-
async placeShort(id, options) {
|
|
122
|
-
return this.locationService.placeByIdShort(id, options);
|
|
123
|
-
}
|
|
124
|
-
async placeFull(id, options) {
|
|
125
|
-
const place = await this.locationService.placeByIdShort(id, options);
|
|
126
|
-
const placeRich = await this.locationService.placeFull(place.url, options);
|
|
127
|
-
return {
|
|
128
|
-
...place,
|
|
129
|
-
...placeRich
|
|
130
|
-
};
|
|
131
|
-
}
|
|
132
|
-
async user(user, options) {
|
|
133
|
-
return this.locationService.user(user, options);
|
|
134
|
-
}
|
|
135
|
-
};
|
|
136
|
-
const locationScraper = new PlacesScraper();
|
|
137
|
-
const atlasObscura = new AtlasObscura(locationScraper);
|
|
138
|
-
|
|
139
|
-
//#endregion
|
|
140
|
-
export { AtlasObscura, atlasObscura };
|
|
141
|
-
//# sourceMappingURL=index.mjs.map
|
package/index.mjs.map
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"index.mjs","names":["USER_AGENTS: string[]","locations: T","place: AOPlaceShort","locationService: PlacesScraper","atlasObscura: AtlasObscura"],"sources":["../src/fetchers/index.ts","../src/utils.ts","../src/vars.ts","../src/services/location.service.ts","../src/index.ts"],"sourcesContent":["const USER_AGENTS: string[] = [\n 'Dalvik/2.1.0 (Linux; U; Android 8.1.0; C6L Build/OPM2.171019.012)',\n 'Mozilla/5.0 (Windows; Windows NT 10.0; x64) AppleWebKit/537.50 (KHTML, like Gecko) Chrome/48.0.2572.385 Safari/537.9 Edge/8.37913',\n 'Mozilla/5.0 (iPhone; CPU iPhone OS 10_3 like Mac OS X) AppleWebKit/602.1.50 (KHTML, like Gecko) Version/10.0 Mobile/14E5239e Safari/602.1',\n 'Dalvik/2.1.0 (Linux; U; Android 11; SM-A505GT Build/RP1A.200720.012)',\n 'Mozilla/5.0 (Windows NT 10.3;; en-US) AppleWebKit/602.25 (KHTML, like Gecko) Chrome/54.0.2457.241 Safari/534.5 Edge/9.65405',\n 'Mozilla/5.0 (Windows; U; Windows NT 10.4; x64) AppleWebKit/536.41 (KHTML, like Gecko) Chrome/53.0.3535.112 Safari/600.5 Edge/14.39786',\n 'Mozilla/5.0 (iPhone; CPU iPhone OS 8_4_0; like Mac OS X) AppleWebKit/601.9 (KHTML, like Gecko) Chrome/47.0.2876.394 Mobile Safari/600.5',\n 'Mozilla/5.0 (iPhone; CPU iPhone OS 7_6_3; like Mac OS X) AppleWebKit/602.24 (KHTML, like Gecko) Chrome/54.0.3204.297 Mobile Safari/537.0',\n 'Mozilla/5.0 (iPhone; CPU iPhone OS 10_9_6; like Mac OS X) AppleWebKit/601.4 (KHTML, like Gecko) Chrome/55.0.2808.105 Mobile Safari/600.4',\n 'Mozilla/5.0 (iPhone; CPU iPhone OS 8_2_1; like Mac OS X) AppleWebKit/600.14 (KHTML, like Gecko) Chrome/50.0.3808.259 Mobile Safari/534.1',\n 'Mozilla/5.0 (iPhone; CPU iPhone OS 10_4_2; like Mac OS X) AppleWebKit/534.21 (KHTML, like Gecko) Chrome/47.0.3531.351 Mobile Safari/533.0'\n];\n\nconst DEFAULT_HEADERS = {\n 'User-Agent': USER_AGENTS[Math.floor(Math.random() * USER_AGENTS.length)],\n 'Cache-Control': 'no-cache',\n};\n\nexport const fetchPage = async (url: string, headersCustom?: Headers): Promise<string> => {\n try {\n const response = await fetch(url, {\n headers: { ...DEFAULT_HEADERS, ...headersCustom },\n credentials: 'omit'\n });\n\n if (!response.ok) {\n throw new Error(`atlas-obscura-api: Bad response ${response.status} for url: ${url}`);\n }\n\n return await response.text();\n } catch (error) {\n console.error('fetchPage error:', error);\n return 'Error';\n }\n};\n","export const parseResults = <T>(matches: RegExpMatchArray): T => {\n let locations: T = null;\n\n if (matches && matches[1]?.length) {\n try {\n locations = JSON.parse(matches[1]);\n } catch (error) {\n console.error('Error parsing all places');\n }\n }\n return locations;\n};\n","export const URL_PLACES_ALL =\n 'https://www.atlasobscura.com/articles/all-places-in-the-atlas-on-one-map';\n\nexport const urlSearch = (coord: { lat: number; lng: number }, page?: number): string =>\n `https://www.atlasobscura.com/search?utf8=%E2%9C%93&q=&location=&nearby=true&lat=${coord.lat\n }&lng=${coord.lng}${page ? '&page=' + page : ''}`;\n\nexport const urlPlaceId = (id: number): string =>\n `https://www.atlasobscura.com/places/${id}.json?place_only=1`;\n\nexport const urlUser = (name: string): string =>\n `https://www.atlasobscura.com/users/${name}`;\n","import { parse } from 'node-html-parser';\nimport { fetchPage } from '../fetchers/index.js';\nimport { AOUser } from '../interfaces/user.interface.js';\nimport { parseResults } from '../utils.js';\nimport { URL_PLACES_ALL, urlPlaceId, urlSearch, urlUser } from '../vars.js';\nimport { AOCoordinates, AOFetchOptions } from './../interfaces/global.js';\nimport { AOLocationResult } from './../interfaces/locations.interface.js';\nimport { AOPlaceExtend, AOPlaceShort } from './../interfaces/placeId.interface.js';\nimport { AOPlacesAll } from './../interfaces/places-all.interface.js';\n\nexport class PlacesScraper {\n public async placesAll(options?: AOFetchOptions): Promise<AOPlacesAll[]> {\n const response = await fetchPage(URL_PLACES_ALL, options?.headers);\n const html = parse(response).innerHTML;\n\n const data = html.match(/AtlasObscura\\.all_places = (.*?);/);\n\n return parseResults<AOPlacesAll[]>(data);\n }\n\n public async search(coords: AOCoordinates, page = 1, options?: AOFetchOptions): Promise<AOLocationResult> {\n const url = urlSearch(coords, page);\n const response = await fetchPage(url, options?.headers);\n const html = parse(response).innerHTML;\n\n const data = html.match(/AtlasObscura\\.place_search = (.*?);/);\n\n return parseResults<AOLocationResult>(data);\n }\n\n public async placeByIdShort(id: number, options?: AOFetchOptions): Promise<AOPlaceShort> {\n const url = urlPlaceId(id);\n const data = await fetchPage(url, options?.headers);\n\n let place: AOPlaceShort = null;\n\n try {\n place = JSON.parse(data);\n if (place && typeof place.hide_from_maps === 'string') {\n (place as any).hide_from_maps = place.hide_from_maps === 'true';\n }\n } catch (error) {\n console.error('Error parsing placeById', id);\n }\n return place;\n }\n\n public async placeFull(url: string, options?: AOFetchOptions): Promise<AOPlaceExtend> {\n const data = await fetchPage(url, options?.headers);\n const html = parse(data);\n\n const descriptions = html.querySelectorAll('.place-body p').map((p) => p.innerHTML);\n\n const sectionDirections = Array.from(html.querySelectorAll('section')).find((sec) => {\n const h2 = sec.querySelector('h2');\n return h2 && h2.textContent.trim() === 'Know Before You Go';\n });\n const directions = sectionDirections\n ? Array.from(sectionDirections.querySelectorAll('p')).map((p) => p.textContent.trim())\n : [];\n\n const tags = html.querySelectorAll('.aon-pill-badge-component').map((x) => {\n return {\n title: x.querySelector('.aon-pill-badge-text')?.innerHTML.trim(),\n link: x.attributes.href\n };\n });\n\n const slides = html.querySelectorAll('.swiper-slide');\n const images = Array.from(slides)\n .map((slide) => {\n const img = slide.querySelector('img');\n return img ? img.getAttribute('src') : null;\n })\n .filter(Boolean);\n\n const imageCover = images[0];\n\n return { description: descriptions, directions, tags, imageCover, images };\n }\n\n public async user(user: string, options?: AOFetchOptions): Promise<AOUser> {\n const url = urlUser(user);\n const response = await fetchPage(url, options?.headers);\n const html = parse(response).innerHTML;\n\n const data = html.match(/AtlasObscura\\.user_profile = (.*?);/);\n\n const usr = parseResults<{ user: AOUser }>(data);\n\n return usr.user;\n }\n}\n","import { AOCoordinates, AOFetchOptions } from './interfaces/global.js';\nimport { AOLocationResult } from './interfaces/locations.interface.js';\nimport { AOPlace, AOPlaceShort } from './interfaces/placeId.interface.js';\nimport { AOPlacesAll } from './interfaces/places-all.interface.js';\nimport { AOUser } from './interfaces/user.interface.js';\nimport { PlacesScraper } from './services/location.service.js';\n\nexport class AtlasObscura {\n constructor(private locationService: PlacesScraper) { }\n\n public async search(coords: AOCoordinates, options?: AOFetchOptions): Promise<AOLocationResult> {\n return this.locationService.search(coords, 1, options);\n }\n\n public async placesAll(options?: AOFetchOptions): Promise<AOPlacesAll[]> {\n return this.locationService.placesAll(options);\n }\n\n public async placeShort(id: number, options?: AOFetchOptions): Promise<AOPlaceShort> {\n return this.locationService.placeByIdShort(id, options);\n }\n\n public async placeFull(id: number, options?: AOFetchOptions): Promise<AOPlace> {\n const place = await this.locationService.placeByIdShort(id, options);\n const placeRich = await this.locationService.placeFull(place.url, options);\n return { ...place, ...placeRich };\n }\n\n public async user(user: string, options?: AOFetchOptions): Promise<AOUser> {\n return this.locationService.user(user, options);\n }\n}\n\nconst locationScraper = new PlacesScraper();\nexport const atlasObscura: AtlasObscura = new AtlasObscura(locationScraper);\nexport type * from './interfaces/index.js';\n"],"mappings":";;;AAAA,MAAMA,cAAwB;CAC5B;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACA;CACD;AAED,MAAM,kBAAkB;CACtB,cAAc,YAAY,KAAK,MAAM,KAAK,QAAQ,GAAG,YAAY,OAAO;CACxE,iBAAiB;CAClB;AAED,MAAa,YAAY,OAAO,KAAa,kBAA6C;AACxF,KAAI;EACF,MAAM,WAAW,MAAM,MAAM,KAAK;GAChC,SAAS;IAAE,GAAG;IAAiB,GAAG;IAAe;GACjD,aAAa;GACd,CAAC;AAEF,MAAI,CAAC,SAAS,GACZ,OAAM,IAAI,MAAM,mCAAmC,SAAS,OAAO,YAAY,MAAM;AAGvF,SAAO,MAAM,SAAS,MAAM;UACrB,OAAO;AACd,UAAQ,MAAM,oBAAoB,MAAM;AACxC,SAAO;;;;;;ACjCX,MAAa,gBAAmB,YAAiC;CAC/D,IAAIC,YAAe;AAEnB,KAAI,WAAW,QAAQ,IAAI,OACzB,KAAI;AACF,cAAY,KAAK,MAAM,QAAQ,GAAG;UAC3B,OAAO;AACd,UAAQ,MAAM,2BAA2B;;AAG7C,QAAO;;;;;ACVT,MAAa,iBACX;AAEF,MAAa,aAAa,OAAqC,SAC7D,mFAAmF,MAAM,IACxF,OAAO,MAAM,MAAM,OAAO,WAAW,OAAO;AAE/C,MAAa,cAAc,OACzB,uCAAuC,GAAG;AAE5C,MAAa,WAAW,SACtB,sCAAsC;;;;ACDxC,IAAa,gBAAb,MAA2B;CACzB,MAAa,UAAU,SAAkD;AAMvE,SAAO,aAJM,MADI,MAAM,UAAU,gBAAgB,SAAS,QAAQ,CACtC,CAAC,UAEX,MAAM,oCAAoC,CAEpB;;CAG1C,MAAa,OAAO,QAAuB,OAAO,GAAG,SAAqD;AAOxG,SAAO,aAJM,MADI,MAAM,UADX,UAAU,QAAQ,KAAK,EACG,SAAS,QAAQ,CAC3B,CAAC,UAEX,MAAM,sCAAsC,CAEnB;;CAG7C,MAAa,eAAe,IAAY,SAAiD;EAEvF,MAAM,OAAO,MAAM,UADP,WAAW,GAAG,EACQ,SAAS,QAAQ;EAEnD,IAAIC,QAAsB;AAE1B,MAAI;AACF,WAAQ,KAAK,MAAM,KAAK;AACxB,OAAI,SAAS,OAAO,MAAM,mBAAmB,SAC3C,CAAC,MAAc,iBAAiB,MAAM,mBAAmB;WAEpD,OAAO;AACd,WAAQ,MAAM,2BAA2B,GAAG;;AAE9C,SAAO;;CAGT,MAAa,UAAU,KAAa,SAAkD;EAEpF,MAAM,OAAO,MADA,MAAM,UAAU,KAAK,SAAS,QAAQ,CAC3B;EAExB,MAAM,eAAe,KAAK,iBAAiB,gBAAgB,CAAC,KAAK,MAAM,EAAE,UAAU;EAEnF,MAAM,oBAAoB,MAAM,KAAK,KAAK,iBAAiB,UAAU,CAAC,CAAC,MAAM,QAAQ;GACnF,MAAM,KAAK,IAAI,cAAc,KAAK;AAClC,UAAO,MAAM,GAAG,YAAY,MAAM,KAAK;IACvC;EACF,MAAM,aAAa,oBACf,MAAM,KAAK,kBAAkB,iBAAiB,IAAI,CAAC,CAAC,KAAK,MAAM,EAAE,YAAY,MAAM,CAAC,GACpF,EAAE;EAEN,MAAM,OAAO,KAAK,iBAAiB,4BAA4B,CAAC,KAAK,MAAM;AACzE,UAAO;IACL,OAAO,EAAE,cAAc,uBAAuB,EAAE,UAAU,MAAM;IAChE,MAAM,EAAE,WAAW;IACpB;IACD;EAEF,MAAM,SAAS,KAAK,iBAAiB,gBAAgB;EACrD,MAAM,SAAS,MAAM,KAAK,OAAO,CAC9B,KAAK,UAAU;GACd,MAAM,MAAM,MAAM,cAAc,MAAM;AACtC,UAAO,MAAM,IAAI,aAAa,MAAM,GAAG;IACvC,CACD,OAAO,QAAQ;AAIlB,SAAO;GAAE,aAAa;GAAc;GAAY;GAAM,YAFnC,OAAO;GAEwC;GAAQ;;CAG5E,MAAa,KAAK,MAAc,SAA2C;AASzE,SAFY,aAJC,MADI,MAAM,UADX,QAAQ,KAAK,EACa,SAAS,QAAQ,CAC3B,CAAC,UAEX,MAAM,sCAAsC,CAEd,CAErC;;;;;;ACnFf,IAAa,eAAb,MAA0B;CACxB,YAAY,AAAQC,iBAAgC;EAAhC;;CAEpB,MAAa,OAAO,QAAuB,SAAqD;AAC9F,SAAO,KAAK,gBAAgB,OAAO,QAAQ,GAAG,QAAQ;;CAGxD,MAAa,UAAU,SAAkD;AACvE,SAAO,KAAK,gBAAgB,UAAU,QAAQ;;CAGhD,MAAa,WAAW,IAAY,SAAiD;AACnF,SAAO,KAAK,gBAAgB,eAAe,IAAI,QAAQ;;CAGzD,MAAa,UAAU,IAAY,SAA4C;EAC7E,MAAM,QAAQ,MAAM,KAAK,gBAAgB,eAAe,IAAI,QAAQ;EACpE,MAAM,YAAY,MAAM,KAAK,gBAAgB,UAAU,MAAM,KAAK,QAAQ;AAC1E,SAAO;GAAE,GAAG;GAAO,GAAG;GAAW;;CAGnC,MAAa,KAAK,MAAc,SAA2C;AACzE,SAAO,KAAK,gBAAgB,KAAK,MAAM,QAAQ;;;AAInD,MAAM,kBAAkB,IAAI,eAAe;AAC3C,MAAaC,eAA6B,IAAI,aAAa,gBAAgB"}
|