@opensite/hooks 2.0.0 → 2.0.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +12 -9
- package/dist/browser/opensite-hooks.umd.cjs +1 -1
- package/dist/browser/opensite-hooks.umd.js +1 -1
- package/dist/browser/opensite-hooks.umd.js.map +1 -1
- package/dist/core/index.cjs +7 -0
- package/dist/core/index.d.ts +13 -0
- package/dist/core/index.js +7 -0
- package/dist/core/useBoolean.cjs +4 -2
- package/dist/core/useBoolean.js +4 -2
- package/dist/core/useMap.cjs +6 -2
- package/dist/core/useMap.js +6 -2
- package/dist/core/useOnClickOutside.cjs +5 -1
- package/dist/core/useOnClickOutside.js +5 -1
- package/dist/core/useOpenGraphExtractor.cjs +59 -0
- package/dist/core/useOpenGraphExtractor.d.ts +64 -0
- package/dist/core/useOpenGraphExtractor.js +59 -0
- package/dist/core/usePrevious.cjs +7 -2
- package/dist/core/usePrevious.js +7 -2
- package/dist/core/useWebsiteExtractorBase.cjs +153 -0
- package/dist/core/useWebsiteExtractorBase.d.ts +9 -0
- package/dist/core/useWebsiteExtractorBase.js +153 -0
- package/dist/core/useWebsiteLinksExtractor.cjs +16 -0
- package/dist/core/useWebsiteLinksExtractor.d.ts +14 -0
- package/dist/core/useWebsiteLinksExtractor.js +16 -0
- package/dist/core/useWebsiteMetaExtractor.cjs +20 -0
- package/dist/core/useWebsiteMetaExtractor.d.ts +12 -0
- package/dist/core/useWebsiteMetaExtractor.js +20 -0
- package/dist/core/useWebsiteRssExtractor.cjs +15 -0
- package/dist/core/useWebsiteRssExtractor.d.ts +12 -0
- package/dist/core/useWebsiteRssExtractor.js +15 -0
- package/dist/core/useWebsiteSchemaExtractor.cjs +15 -0
- package/dist/core/useWebsiteSchemaExtractor.d.ts +11 -0
- package/dist/core/useWebsiteSchemaExtractor.js +15 -0
- package/dist/core/websiteExtractorService.cjs +66 -0
- package/dist/core/websiteExtractorService.d.ts +3 -0
- package/dist/core/websiteExtractorService.js +66 -0
- package/dist/core/websiteExtractorTypes.cjs +25 -0
- package/dist/core/websiteExtractorTypes.d.ts +65 -0
- package/dist/core/websiteExtractorTypes.js +25 -0
- package/package.json +36 -1
- package/dist/hooks/index.cjs +0 -16
- package/dist/hooks/index.d.ts +0 -24
- package/dist/hooks/index.js +0 -16
- package/dist/hooks/useBoolean.cjs +0 -1
- package/dist/hooks/useBoolean.d.ts +0 -2
- package/dist/hooks/useBoolean.js +0 -1
- package/dist/hooks/useCopyToClipboard.cjs +0 -1
- package/dist/hooks/useCopyToClipboard.d.ts +0 -2
- package/dist/hooks/useCopyToClipboard.js +0 -1
- package/dist/hooks/useDebounceCallback.cjs +0 -1
- package/dist/hooks/useDebounceCallback.d.ts +0 -2
- package/dist/hooks/useDebounceCallback.js +0 -1
- package/dist/hooks/useDebounceValue.cjs +0 -1
- package/dist/hooks/useDebounceValue.d.ts +0 -2
- package/dist/hooks/useDebounceValue.js +0 -1
- package/dist/hooks/useEventListener.cjs +0 -1
- package/dist/hooks/useEventListener.d.ts +0 -1
- package/dist/hooks/useEventListener.js +0 -1
- package/dist/hooks/useHover.cjs +0 -1
- package/dist/hooks/useHover.d.ts +0 -1
- package/dist/hooks/useHover.js +0 -1
- package/dist/hooks/useIsClient.cjs +0 -1
- package/dist/hooks/useIsClient.d.ts +0 -1
- package/dist/hooks/useIsClient.js +0 -1
- package/dist/hooks/useIsomorphicLayoutEffect.cjs +0 -1
- package/dist/hooks/useIsomorphicLayoutEffect.d.ts +0 -1
- package/dist/hooks/useIsomorphicLayoutEffect.js +0 -1
- package/dist/hooks/useLocalStorage.cjs +0 -1
- package/dist/hooks/useLocalStorage.d.ts +0 -2
- package/dist/hooks/useLocalStorage.js +0 -1
- package/dist/hooks/useMap.cjs +0 -1
- package/dist/hooks/useMap.d.ts +0 -2
- package/dist/hooks/useMap.js +0 -1
- package/dist/hooks/useMediaQuery.cjs +0 -1
- package/dist/hooks/useMediaQuery.d.ts +0 -2
- package/dist/hooks/useMediaQuery.js +0 -1
- package/dist/hooks/useOnClickOutside.cjs +0 -1
- package/dist/hooks/useOnClickOutside.d.ts +0 -1
- package/dist/hooks/useOnClickOutside.js +0 -1
- package/dist/hooks/usePrevious.cjs +0 -1
- package/dist/hooks/usePrevious.d.ts +0 -1
- package/dist/hooks/usePrevious.js +0 -1
- package/dist/hooks/useResizeObserver.cjs +0 -1
- package/dist/hooks/useResizeObserver.d.ts +0 -1
- package/dist/hooks/useResizeObserver.js +0 -1
- package/dist/hooks/useSessionStorage.cjs +0 -1
- package/dist/hooks/useSessionStorage.d.ts +0 -2
- package/dist/hooks/useSessionStorage.js +0 -1
- package/dist/hooks/useThrottle.cjs +0 -1
- package/dist/hooks/useThrottle.d.ts +0 -2
- package/dist/hooks/useThrottle.js +0 -1
- package/dist/test/setup.cjs +0 -98
- package/dist/test/setup.d.ts +0 -5
- package/dist/test/setup.js +0 -98
- package/dist/test/utils.cjs +0 -73
- package/dist/test/utils.js +0 -73
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { useCallback, useMemo } from "react";
|
|
2
|
+
import { useWebsiteExtractorBase } from "./useWebsiteExtractorBase.js";
|
|
3
|
+
const DEFAULT_SKIP_PATTERNS = [
|
|
4
|
+
/search\.google\.com\/local\/reviews/i,
|
|
5
|
+
/google\.com\/maps\/place/i,
|
|
6
|
+
/maps\.google\.com/i,
|
|
7
|
+
/opentable\.com/i,
|
|
8
|
+
];
|
|
9
|
+
const pickFirstString = (...values) => {
|
|
10
|
+
for (const value of values) {
|
|
11
|
+
if (typeof value === "string" && value.trim().length > 0) {
|
|
12
|
+
return value;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
return undefined;
|
|
16
|
+
};
|
|
17
|
+
const safeHost = (value) => {
|
|
18
|
+
if (!value) {
|
|
19
|
+
return undefined;
|
|
20
|
+
}
|
|
21
|
+
try {
|
|
22
|
+
return new URL(value).hostname;
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return undefined;
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
export function useOpenGraphExtractor(options) {
|
|
29
|
+
const skipPatterns = useMemo(() => options.skipPatterns ?? DEFAULT_SKIP_PATTERNS, [options.skipPatterns]);
|
|
30
|
+
const shouldSkip = useCallback((url) => skipPatterns.some((pattern) => pattern.test(url)), [skipPatterns]);
|
|
31
|
+
const selectData = useCallback((payload, _raw, meta) => {
|
|
32
|
+
const { openGraph, htmlInferred, hybridGraph } = payload;
|
|
33
|
+
const description = pickFirstString(openGraph?.description, hybridGraph?.description, htmlInferred?.description);
|
|
34
|
+
const title = pickFirstString(openGraph?.title, hybridGraph?.title, htmlInferred?.title);
|
|
35
|
+
const siteName = pickFirstString(openGraph?.site_name, hybridGraph?.site_name, htmlInferred?.site_name);
|
|
36
|
+
const favicon = pickFirstString(hybridGraph?.favicon, htmlInferred?.favicon);
|
|
37
|
+
const image = pickFirstString(openGraph?.image?.url ?? undefined, hybridGraph?.image ?? undefined, htmlInferred?.image ?? undefined, htmlInferred?.images?.[0]);
|
|
38
|
+
const video = pickFirstString(openGraph?.video?.url ?? undefined, hybridGraph?.video ?? undefined);
|
|
39
|
+
const videoType = pickFirstString(hybridGraph?.videoType, htmlInferred?.videoType);
|
|
40
|
+
const resolvedUrl = pickFirstString(meta.url, openGraph?.url ?? undefined, hybridGraph?.url ?? undefined, htmlInferred?.url ?? undefined, meta.finalUrl, meta.normalizedUrl, meta.requestedUrl) ?? "";
|
|
41
|
+
return {
|
|
42
|
+
description,
|
|
43
|
+
favicon,
|
|
44
|
+
image,
|
|
45
|
+
video,
|
|
46
|
+
videoType,
|
|
47
|
+
siteName,
|
|
48
|
+
title,
|
|
49
|
+
url: resolvedUrl,
|
|
50
|
+
siteHost: safeHost(resolvedUrl),
|
|
51
|
+
};
|
|
52
|
+
}, []);
|
|
53
|
+
return useWebsiteExtractorBase({
|
|
54
|
+
endpoint: "open-graph",
|
|
55
|
+
options,
|
|
56
|
+
selectData,
|
|
57
|
+
shouldSkip,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import type { WebsiteExtractorOptions, WebsiteExtractorResponse, WebsiteExtractorResult } from "./websiteExtractorTypes.js";
|
|
2
|
+
export interface OpenGraphImage {
|
|
3
|
+
url?: string | null;
|
|
4
|
+
height?: string | null;
|
|
5
|
+
width?: string | null;
|
|
6
|
+
}
|
|
7
|
+
export interface OpenGraphVideo {
|
|
8
|
+
url?: string | null;
|
|
9
|
+
height?: string | null;
|
|
10
|
+
width?: string | null;
|
|
11
|
+
}
|
|
12
|
+
export interface OpenGraphData {
|
|
13
|
+
description?: string | null;
|
|
14
|
+
title?: string | null;
|
|
15
|
+
site_name?: string | null;
|
|
16
|
+
image?: OpenGraphImage | null;
|
|
17
|
+
video?: OpenGraphVideo | null;
|
|
18
|
+
url?: string | null;
|
|
19
|
+
ogType?: string | null;
|
|
20
|
+
}
|
|
21
|
+
export interface OpenGraphHtmlInferredData {
|
|
22
|
+
description?: string | null;
|
|
23
|
+
title?: string | null;
|
|
24
|
+
type?: string | null;
|
|
25
|
+
videoType?: string | null;
|
|
26
|
+
url?: string | null;
|
|
27
|
+
favicon?: string | null;
|
|
28
|
+
images?: string[] | null;
|
|
29
|
+
image?: string | null;
|
|
30
|
+
site_name?: string | null;
|
|
31
|
+
}
|
|
32
|
+
export interface OpenGraphHybridData {
|
|
33
|
+
description?: string | null;
|
|
34
|
+
title?: string | null;
|
|
35
|
+
type?: string | null;
|
|
36
|
+
image?: string | null;
|
|
37
|
+
video?: string | null;
|
|
38
|
+
videoType?: string | null;
|
|
39
|
+
favicon?: string | null;
|
|
40
|
+
site_name?: string | null;
|
|
41
|
+
url?: string | null;
|
|
42
|
+
videoWidth?: number | null;
|
|
43
|
+
videoHeight?: number | null;
|
|
44
|
+
}
|
|
45
|
+
export type OpenGraphResponse = WebsiteExtractorResponse<{
|
|
46
|
+
openGraph: OpenGraphData;
|
|
47
|
+
htmlInferred: OpenGraphHtmlInferredData;
|
|
48
|
+
hybridGraph: OpenGraphHybridData;
|
|
49
|
+
}>;
|
|
50
|
+
export interface OpenGraphSummary {
|
|
51
|
+
description?: string;
|
|
52
|
+
favicon?: string;
|
|
53
|
+
image?: string;
|
|
54
|
+
video?: string;
|
|
55
|
+
videoType?: string;
|
|
56
|
+
siteName?: string;
|
|
57
|
+
title?: string;
|
|
58
|
+
url: string;
|
|
59
|
+
siteHost?: string;
|
|
60
|
+
}
|
|
61
|
+
export interface OpenGraphExtractorOptions extends WebsiteExtractorOptions {
|
|
62
|
+
skipPatterns?: RegExp[];
|
|
63
|
+
}
|
|
64
|
+
export declare function useOpenGraphExtractor(options: OpenGraphExtractorOptions): WebsiteExtractorResult<OpenGraphSummary, OpenGraphResponse>;
|
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
import { useCallback, useMemo } from "react";
|
|
2
|
+
import { useWebsiteExtractorBase } from "./useWebsiteExtractorBase.js";
|
|
3
|
+
const DEFAULT_SKIP_PATTERNS = [
|
|
4
|
+
/search\.google\.com\/local\/reviews/i,
|
|
5
|
+
/google\.com\/maps\/place/i,
|
|
6
|
+
/maps\.google\.com/i,
|
|
7
|
+
/opentable\.com/i,
|
|
8
|
+
];
|
|
9
|
+
const pickFirstString = (...values) => {
|
|
10
|
+
for (const value of values) {
|
|
11
|
+
if (typeof value === "string" && value.trim().length > 0) {
|
|
12
|
+
return value;
|
|
13
|
+
}
|
|
14
|
+
}
|
|
15
|
+
return undefined;
|
|
16
|
+
};
|
|
17
|
+
const safeHost = (value) => {
|
|
18
|
+
if (!value) {
|
|
19
|
+
return undefined;
|
|
20
|
+
}
|
|
21
|
+
try {
|
|
22
|
+
return new URL(value).hostname;
|
|
23
|
+
}
|
|
24
|
+
catch {
|
|
25
|
+
return undefined;
|
|
26
|
+
}
|
|
27
|
+
};
|
|
28
|
+
export function useOpenGraphExtractor(options) {
|
|
29
|
+
const skipPatterns = useMemo(() => options.skipPatterns ?? DEFAULT_SKIP_PATTERNS, [options.skipPatterns]);
|
|
30
|
+
const shouldSkip = useCallback((url) => skipPatterns.some((pattern) => pattern.test(url)), [skipPatterns]);
|
|
31
|
+
const selectData = useCallback((payload, _raw, meta) => {
|
|
32
|
+
const { openGraph, htmlInferred, hybridGraph } = payload;
|
|
33
|
+
const description = pickFirstString(openGraph?.description, hybridGraph?.description, htmlInferred?.description);
|
|
34
|
+
const title = pickFirstString(openGraph?.title, hybridGraph?.title, htmlInferred?.title);
|
|
35
|
+
const siteName = pickFirstString(openGraph?.site_name, hybridGraph?.site_name, htmlInferred?.site_name);
|
|
36
|
+
const favicon = pickFirstString(hybridGraph?.favicon, htmlInferred?.favicon);
|
|
37
|
+
const image = pickFirstString(openGraph?.image?.url ?? undefined, hybridGraph?.image ?? undefined, htmlInferred?.image ?? undefined, htmlInferred?.images?.[0]);
|
|
38
|
+
const video = pickFirstString(openGraph?.video?.url ?? undefined, hybridGraph?.video ?? undefined);
|
|
39
|
+
const videoType = pickFirstString(hybridGraph?.videoType, htmlInferred?.videoType);
|
|
40
|
+
const resolvedUrl = pickFirstString(meta.url, openGraph?.url ?? undefined, hybridGraph?.url ?? undefined, htmlInferred?.url ?? undefined, meta.finalUrl, meta.normalizedUrl, meta.requestedUrl) ?? "";
|
|
41
|
+
return {
|
|
42
|
+
description,
|
|
43
|
+
favicon,
|
|
44
|
+
image,
|
|
45
|
+
video,
|
|
46
|
+
videoType,
|
|
47
|
+
siteName,
|
|
48
|
+
title,
|
|
49
|
+
url: resolvedUrl,
|
|
50
|
+
siteHost: safeHost(resolvedUrl),
|
|
51
|
+
};
|
|
52
|
+
}, []);
|
|
53
|
+
return useWebsiteExtractorBase({
|
|
54
|
+
endpoint: "open-graph",
|
|
55
|
+
options,
|
|
56
|
+
selectData,
|
|
57
|
+
shouldSkip,
|
|
58
|
+
});
|
|
59
|
+
}
|
|
@@ -1,7 +1,12 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { useRef } from "react";
|
|
2
|
+
import { useIsomorphicLayoutEffect } from "./useIsomorphicLayoutEffect.js";
|
|
2
3
|
export function usePrevious(value) {
|
|
3
4
|
const ref = useRef();
|
|
4
|
-
|
|
5
|
+
// Use useIsomorphicLayoutEffect to capture the previous value synchronously
|
|
6
|
+
// BEFORE paint. This ensures that during render, ref.current holds the actual
|
|
7
|
+
// previous value (from the last render), not the current value.
|
|
8
|
+
// Using useEffect would update AFTER paint, making comparisons incorrect.
|
|
9
|
+
useIsomorphicLayoutEffect(() => {
|
|
5
10
|
ref.current = value;
|
|
6
11
|
}, [value]);
|
|
7
12
|
return ref.current;
|
package/dist/core/usePrevious.js
CHANGED
|
@@ -1,7 +1,12 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { useRef } from "react";
|
|
2
|
+
import { useIsomorphicLayoutEffect } from "./useIsomorphicLayoutEffect.js";
|
|
2
3
|
export function usePrevious(value) {
|
|
3
4
|
const ref = useRef();
|
|
4
|
-
|
|
5
|
+
// Use useIsomorphicLayoutEffect to capture the previous value synchronously
|
|
6
|
+
// BEFORE paint. This ensures that during render, ref.current holds the actual
|
|
7
|
+
// previous value (from the last render), not the current value.
|
|
8
|
+
// Using useEffect would update AFTER paint, making comparisons incorrect.
|
|
9
|
+
useIsomorphicLayoutEffect(() => {
|
|
5
10
|
ref.current = value;
|
|
6
11
|
}, [value]);
|
|
7
12
|
return ref.current;
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
|
2
|
+
import { useDebounceCallback } from "./useDebounceCallback.js";
|
|
3
|
+
import { useDebounceValue } from "./useDebounceValue.js";
|
|
4
|
+
import { useIsClient } from "./useIsClient.js";
|
|
5
|
+
import { useMap } from "./useMap.js";
|
|
6
|
+
import { fetchWebsiteExtractor } from "./websiteExtractorService.js";
|
|
7
|
+
import { DEFAULT_EXTRACTOR_CACHE, DEFAULT_EXTRACTOR_DEBOUNCE_MS, DEFAULT_EXTRACTOR_ENABLED, DEFAULT_EXTRACTOR_REFRESH_DEBOUNCE_MS, DEFAULT_WEBSITE_EXTRACTOR_BASE_URL, extractWebsiteMeta, stripWebsiteMeta, } from "./websiteExtractorTypes.js";
|
|
8
|
+
export function useWebsiteExtractorBase(config) {
|
|
9
|
+
const { endpoint, options, selectData, shouldSkip } = config;
|
|
10
|
+
const isClient = useIsClient();
|
|
11
|
+
const [state, setState] = useState({
|
|
12
|
+
loading: false,
|
|
13
|
+
});
|
|
14
|
+
const [, cacheActions] = useMap();
|
|
15
|
+
const cacheEnabled = options.cache ?? DEFAULT_EXTRACTOR_CACHE;
|
|
16
|
+
const enabled = options.enabled ?? DEFAULT_EXTRACTOR_ENABLED;
|
|
17
|
+
const debounceMs = options.debounceMs ?? DEFAULT_EXTRACTOR_DEBOUNCE_MS;
|
|
18
|
+
const refreshDebounceMs = options.refreshDebounceMs ?? DEFAULT_EXTRACTOR_REFRESH_DEBOUNCE_MS;
|
|
19
|
+
const normalizedUrl = useMemo(() => {
|
|
20
|
+
return options.url?.trim() ?? "";
|
|
21
|
+
}, [options.url]);
|
|
22
|
+
const debouncedUrl = useDebounceValue(normalizedUrl, debounceMs);
|
|
23
|
+
const refreshCounterRef = useRef(0);
|
|
24
|
+
const lastRefreshHandledRef = useRef(0);
|
|
25
|
+
const [refreshToken, setRefreshToken] = useState(0);
|
|
26
|
+
const { debouncedCallback: scheduleRefresh, cancel: cancelRefresh } = useDebounceCallback(() => {
|
|
27
|
+
refreshCounterRef.current += 1;
|
|
28
|
+
setRefreshToken(refreshCounterRef.current);
|
|
29
|
+
}, refreshDebounceMs);
|
|
30
|
+
const refresh = useCallback(() => {
|
|
31
|
+
scheduleRefresh();
|
|
32
|
+
}, [scheduleRefresh]);
|
|
33
|
+
useEffect(() => () => cancelRefresh(), [cancelRefresh]);
|
|
34
|
+
const requestKey = useMemo(() => {
|
|
35
|
+
if (!debouncedUrl) {
|
|
36
|
+
return "";
|
|
37
|
+
}
|
|
38
|
+
const baseUrl = options.baseUrl ?? DEFAULT_WEBSITE_EXTRACTOR_BASE_URL;
|
|
39
|
+
const apiKey = options.apiKey ?? "";
|
|
40
|
+
return `${endpoint}:${baseUrl}:${apiKey}:${debouncedUrl}`;
|
|
41
|
+
}, [
|
|
42
|
+
endpoint,
|
|
43
|
+
options.apiKey,
|
|
44
|
+
options.baseUrl,
|
|
45
|
+
debouncedUrl,
|
|
46
|
+
]);
|
|
47
|
+
const inFlightControllerRef = useRef(null);
|
|
48
|
+
useEffect(() => {
|
|
49
|
+
if (!isClient) {
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
if (!enabled || !debouncedUrl) {
|
|
53
|
+
setState({ loading: false });
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
if (shouldSkip?.(debouncedUrl)) {
|
|
57
|
+
setState({ loading: false });
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const forceRefresh = refreshToken !== lastRefreshHandledRef.current;
|
|
61
|
+
if (forceRefresh) {
|
|
62
|
+
lastRefreshHandledRef.current = refreshToken;
|
|
63
|
+
}
|
|
64
|
+
if (cacheEnabled && !forceRefresh) {
|
|
65
|
+
const cached = cacheActions.get(requestKey);
|
|
66
|
+
if (cached) {
|
|
67
|
+
setState({
|
|
68
|
+
loading: false,
|
|
69
|
+
data: cached.data,
|
|
70
|
+
raw: cached.raw,
|
|
71
|
+
meta: cached.meta,
|
|
72
|
+
});
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
inFlightControllerRef.current?.abort();
|
|
77
|
+
const controller = new AbortController();
|
|
78
|
+
inFlightControllerRef.current = controller;
|
|
79
|
+
setState((prev) => ({
|
|
80
|
+
...prev,
|
|
81
|
+
loading: true,
|
|
82
|
+
error: undefined,
|
|
83
|
+
}));
|
|
84
|
+
fetchWebsiteExtractor({
|
|
85
|
+
endpoint,
|
|
86
|
+
url: debouncedUrl,
|
|
87
|
+
apiKey: options.apiKey,
|
|
88
|
+
baseUrl: options.baseUrl,
|
|
89
|
+
signal: controller.signal,
|
|
90
|
+
})
|
|
91
|
+
.then((result) => {
|
|
92
|
+
if (controller.signal.aborted) {
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
if (!result.ok) {
|
|
96
|
+
setState((prev) => ({
|
|
97
|
+
...prev,
|
|
98
|
+
loading: false,
|
|
99
|
+
error: result.error,
|
|
100
|
+
}));
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
const raw = result.response;
|
|
104
|
+
const meta = extractWebsiteMeta(raw);
|
|
105
|
+
const payload = stripWebsiteMeta(raw);
|
|
106
|
+
const data = selectData(payload, raw, meta);
|
|
107
|
+
if (cacheEnabled) {
|
|
108
|
+
cacheActions.set(requestKey, { data, raw, meta });
|
|
109
|
+
}
|
|
110
|
+
setState({
|
|
111
|
+
loading: false,
|
|
112
|
+
data,
|
|
113
|
+
raw,
|
|
114
|
+
meta,
|
|
115
|
+
});
|
|
116
|
+
})
|
|
117
|
+
.catch((error) => {
|
|
118
|
+
if (controller.signal.aborted) {
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
setState((prev) => ({
|
|
122
|
+
...prev,
|
|
123
|
+
loading: false,
|
|
124
|
+
error: {
|
|
125
|
+
message: error instanceof Error
|
|
126
|
+
? error.message
|
|
127
|
+
: "Request failed unexpectedly.",
|
|
128
|
+
raw: error,
|
|
129
|
+
},
|
|
130
|
+
}));
|
|
131
|
+
});
|
|
132
|
+
return () => {
|
|
133
|
+
controller.abort();
|
|
134
|
+
};
|
|
135
|
+
}, [
|
|
136
|
+
cacheActions,
|
|
137
|
+
cacheEnabled,
|
|
138
|
+
debouncedUrl,
|
|
139
|
+
enabled,
|
|
140
|
+
endpoint,
|
|
141
|
+
isClient,
|
|
142
|
+
options.apiKey,
|
|
143
|
+
options.baseUrl,
|
|
144
|
+
requestKey,
|
|
145
|
+
refreshToken,
|
|
146
|
+
selectData,
|
|
147
|
+
shouldSkip,
|
|
148
|
+
]);
|
|
149
|
+
return useMemo(() => ({
|
|
150
|
+
...state,
|
|
151
|
+
refresh,
|
|
152
|
+
}), [refresh, state]);
|
|
153
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import { type WebsiteExtractorOptions, type WebsiteExtractorResult, type WebsiteExtractMeta } from "./websiteExtractorTypes.js";
|
|
2
|
+
interface UseWebsiteExtractorBaseConfig<TResponse extends WebsiteExtractMeta, TData> {
|
|
3
|
+
endpoint: string;
|
|
4
|
+
options: WebsiteExtractorOptions;
|
|
5
|
+
selectData: (payload: Omit<TResponse, keyof WebsiteExtractMeta>, raw: TResponse, meta: WebsiteExtractMeta) => TData;
|
|
6
|
+
shouldSkip?: (url: string) => boolean;
|
|
7
|
+
}
|
|
8
|
+
export declare function useWebsiteExtractorBase<TResponse extends WebsiteExtractMeta, TData>(config: UseWebsiteExtractorBaseConfig<TResponse, TData>): WebsiteExtractorResult<TData, TResponse>;
|
|
9
|
+
export {};
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
import { useCallback, useEffect, useMemo, useRef, useState } from "react";
|
|
2
|
+
import { useDebounceCallback } from "./useDebounceCallback.js";
|
|
3
|
+
import { useDebounceValue } from "./useDebounceValue.js";
|
|
4
|
+
import { useIsClient } from "./useIsClient.js";
|
|
5
|
+
import { useMap } from "./useMap.js";
|
|
6
|
+
import { fetchWebsiteExtractor } from "./websiteExtractorService.js";
|
|
7
|
+
import { DEFAULT_EXTRACTOR_CACHE, DEFAULT_EXTRACTOR_DEBOUNCE_MS, DEFAULT_EXTRACTOR_ENABLED, DEFAULT_EXTRACTOR_REFRESH_DEBOUNCE_MS, DEFAULT_WEBSITE_EXTRACTOR_BASE_URL, extractWebsiteMeta, stripWebsiteMeta, } from "./websiteExtractorTypes.js";
|
|
8
|
+
export function useWebsiteExtractorBase(config) {
|
|
9
|
+
const { endpoint, options, selectData, shouldSkip } = config;
|
|
10
|
+
const isClient = useIsClient();
|
|
11
|
+
const [state, setState] = useState({
|
|
12
|
+
loading: false,
|
|
13
|
+
});
|
|
14
|
+
const [, cacheActions] = useMap();
|
|
15
|
+
const cacheEnabled = options.cache ?? DEFAULT_EXTRACTOR_CACHE;
|
|
16
|
+
const enabled = options.enabled ?? DEFAULT_EXTRACTOR_ENABLED;
|
|
17
|
+
const debounceMs = options.debounceMs ?? DEFAULT_EXTRACTOR_DEBOUNCE_MS;
|
|
18
|
+
const refreshDebounceMs = options.refreshDebounceMs ?? DEFAULT_EXTRACTOR_REFRESH_DEBOUNCE_MS;
|
|
19
|
+
const normalizedUrl = useMemo(() => {
|
|
20
|
+
return options.url?.trim() ?? "";
|
|
21
|
+
}, [options.url]);
|
|
22
|
+
const debouncedUrl = useDebounceValue(normalizedUrl, debounceMs);
|
|
23
|
+
const refreshCounterRef = useRef(0);
|
|
24
|
+
const lastRefreshHandledRef = useRef(0);
|
|
25
|
+
const [refreshToken, setRefreshToken] = useState(0);
|
|
26
|
+
const { debouncedCallback: scheduleRefresh, cancel: cancelRefresh } = useDebounceCallback(() => {
|
|
27
|
+
refreshCounterRef.current += 1;
|
|
28
|
+
setRefreshToken(refreshCounterRef.current);
|
|
29
|
+
}, refreshDebounceMs);
|
|
30
|
+
const refresh = useCallback(() => {
|
|
31
|
+
scheduleRefresh();
|
|
32
|
+
}, [scheduleRefresh]);
|
|
33
|
+
useEffect(() => () => cancelRefresh(), [cancelRefresh]);
|
|
34
|
+
const requestKey = useMemo(() => {
|
|
35
|
+
if (!debouncedUrl) {
|
|
36
|
+
return "";
|
|
37
|
+
}
|
|
38
|
+
const baseUrl = options.baseUrl ?? DEFAULT_WEBSITE_EXTRACTOR_BASE_URL;
|
|
39
|
+
const apiKey = options.apiKey ?? "";
|
|
40
|
+
return `${endpoint}:${baseUrl}:${apiKey}:${debouncedUrl}`;
|
|
41
|
+
}, [
|
|
42
|
+
endpoint,
|
|
43
|
+
options.apiKey,
|
|
44
|
+
options.baseUrl,
|
|
45
|
+
debouncedUrl,
|
|
46
|
+
]);
|
|
47
|
+
const inFlightControllerRef = useRef(null);
|
|
48
|
+
useEffect(() => {
|
|
49
|
+
if (!isClient) {
|
|
50
|
+
return;
|
|
51
|
+
}
|
|
52
|
+
if (!enabled || !debouncedUrl) {
|
|
53
|
+
setState({ loading: false });
|
|
54
|
+
return;
|
|
55
|
+
}
|
|
56
|
+
if (shouldSkip?.(debouncedUrl)) {
|
|
57
|
+
setState({ loading: false });
|
|
58
|
+
return;
|
|
59
|
+
}
|
|
60
|
+
const forceRefresh = refreshToken !== lastRefreshHandledRef.current;
|
|
61
|
+
if (forceRefresh) {
|
|
62
|
+
lastRefreshHandledRef.current = refreshToken;
|
|
63
|
+
}
|
|
64
|
+
if (cacheEnabled && !forceRefresh) {
|
|
65
|
+
const cached = cacheActions.get(requestKey);
|
|
66
|
+
if (cached) {
|
|
67
|
+
setState({
|
|
68
|
+
loading: false,
|
|
69
|
+
data: cached.data,
|
|
70
|
+
raw: cached.raw,
|
|
71
|
+
meta: cached.meta,
|
|
72
|
+
});
|
|
73
|
+
return;
|
|
74
|
+
}
|
|
75
|
+
}
|
|
76
|
+
inFlightControllerRef.current?.abort();
|
|
77
|
+
const controller = new AbortController();
|
|
78
|
+
inFlightControllerRef.current = controller;
|
|
79
|
+
setState((prev) => ({
|
|
80
|
+
...prev,
|
|
81
|
+
loading: true,
|
|
82
|
+
error: undefined,
|
|
83
|
+
}));
|
|
84
|
+
fetchWebsiteExtractor({
|
|
85
|
+
endpoint,
|
|
86
|
+
url: debouncedUrl,
|
|
87
|
+
apiKey: options.apiKey,
|
|
88
|
+
baseUrl: options.baseUrl,
|
|
89
|
+
signal: controller.signal,
|
|
90
|
+
})
|
|
91
|
+
.then((result) => {
|
|
92
|
+
if (controller.signal.aborted) {
|
|
93
|
+
return;
|
|
94
|
+
}
|
|
95
|
+
if (!result.ok) {
|
|
96
|
+
setState((prev) => ({
|
|
97
|
+
...prev,
|
|
98
|
+
loading: false,
|
|
99
|
+
error: result.error,
|
|
100
|
+
}));
|
|
101
|
+
return;
|
|
102
|
+
}
|
|
103
|
+
const raw = result.response;
|
|
104
|
+
const meta = extractWebsiteMeta(raw);
|
|
105
|
+
const payload = stripWebsiteMeta(raw);
|
|
106
|
+
const data = selectData(payload, raw, meta);
|
|
107
|
+
if (cacheEnabled) {
|
|
108
|
+
cacheActions.set(requestKey, { data, raw, meta });
|
|
109
|
+
}
|
|
110
|
+
setState({
|
|
111
|
+
loading: false,
|
|
112
|
+
data,
|
|
113
|
+
raw,
|
|
114
|
+
meta,
|
|
115
|
+
});
|
|
116
|
+
})
|
|
117
|
+
.catch((error) => {
|
|
118
|
+
if (controller.signal.aborted) {
|
|
119
|
+
return;
|
|
120
|
+
}
|
|
121
|
+
setState((prev) => ({
|
|
122
|
+
...prev,
|
|
123
|
+
loading: false,
|
|
124
|
+
error: {
|
|
125
|
+
message: error instanceof Error
|
|
126
|
+
? error.message
|
|
127
|
+
: "Request failed unexpectedly.",
|
|
128
|
+
raw: error,
|
|
129
|
+
},
|
|
130
|
+
}));
|
|
131
|
+
});
|
|
132
|
+
return () => {
|
|
133
|
+
controller.abort();
|
|
134
|
+
};
|
|
135
|
+
}, [
|
|
136
|
+
cacheActions,
|
|
137
|
+
cacheEnabled,
|
|
138
|
+
debouncedUrl,
|
|
139
|
+
enabled,
|
|
140
|
+
endpoint,
|
|
141
|
+
isClient,
|
|
142
|
+
options.apiKey,
|
|
143
|
+
options.baseUrl,
|
|
144
|
+
requestKey,
|
|
145
|
+
refreshToken,
|
|
146
|
+
selectData,
|
|
147
|
+
shouldSkip,
|
|
148
|
+
]);
|
|
149
|
+
return useMemo(() => ({
|
|
150
|
+
...state,
|
|
151
|
+
refresh,
|
|
152
|
+
}), [refresh, state]);
|
|
153
|
+
}
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { useCallback } from "react";
|
|
2
|
+
import { useWebsiteExtractorBase } from "./useWebsiteExtractorBase.js";
|
|
3
|
+
export function useWebsiteLinksExtractor(options) {
|
|
4
|
+
const selectData = useCallback((payload) => {
|
|
5
|
+
return {
|
|
6
|
+
totalLinks: payload.totalLinks ?? 0,
|
|
7
|
+
uniqueDomains: payload.uniqueDomains ?? 0,
|
|
8
|
+
links: payload.links ?? [],
|
|
9
|
+
};
|
|
10
|
+
}, []);
|
|
11
|
+
return useWebsiteExtractorBase({
|
|
12
|
+
endpoint: "links",
|
|
13
|
+
options,
|
|
14
|
+
selectData,
|
|
15
|
+
});
|
|
16
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import type { WebsiteExtractorOptions, WebsiteExtractorResponse, WebsiteExtractorResult } from "./websiteExtractorTypes.js";
|
|
2
|
+
export interface WebsiteLinkRecord {
|
|
3
|
+
url: string;
|
|
4
|
+
text: string;
|
|
5
|
+
isExternal: boolean;
|
|
6
|
+
domain?: string | null;
|
|
7
|
+
}
|
|
8
|
+
export interface WebsiteLinksPayload {
|
|
9
|
+
totalLinks: number;
|
|
10
|
+
uniqueDomains: number;
|
|
11
|
+
links: WebsiteLinkRecord[];
|
|
12
|
+
}
|
|
13
|
+
export type WebsiteLinksResponse = WebsiteExtractorResponse<WebsiteLinksPayload>;
|
|
14
|
+
export declare function useWebsiteLinksExtractor(options: WebsiteExtractorOptions): WebsiteExtractorResult<WebsiteLinksPayload, WebsiteLinksResponse>;
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
import { useCallback } from "react";
|
|
2
|
+
import { useWebsiteExtractorBase } from "./useWebsiteExtractorBase.js";
|
|
3
|
+
export function useWebsiteLinksExtractor(options) {
|
|
4
|
+
const selectData = useCallback((payload) => {
|
|
5
|
+
return {
|
|
6
|
+
totalLinks: payload.totalLinks ?? 0,
|
|
7
|
+
uniqueDomains: payload.uniqueDomains ?? 0,
|
|
8
|
+
links: payload.links ?? [],
|
|
9
|
+
};
|
|
10
|
+
}, []);
|
|
11
|
+
return useWebsiteExtractorBase({
|
|
12
|
+
endpoint: "links",
|
|
13
|
+
options,
|
|
14
|
+
selectData,
|
|
15
|
+
});
|
|
16
|
+
}
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { useCallback } from "react";
|
|
2
|
+
import { useWebsiteExtractorBase } from "./useWebsiteExtractorBase.js";
|
|
3
|
+
export function useWebsiteMetaExtractor(options) {
|
|
4
|
+
const selectData = useCallback((payload) => {
|
|
5
|
+
return {
|
|
6
|
+
title: payload.title ?? undefined,
|
|
7
|
+
description: payload.description ?? undefined,
|
|
8
|
+
language: payload.language ?? undefined,
|
|
9
|
+
canonicalUrl: payload.canonicalUrl ?? undefined,
|
|
10
|
+
feedUrl: payload.feedUrl ?? null,
|
|
11
|
+
textContentLength: payload.textContentLength ?? undefined,
|
|
12
|
+
metaTags: payload.metaTags ?? {},
|
|
13
|
+
};
|
|
14
|
+
}, []);
|
|
15
|
+
return useWebsiteExtractorBase({
|
|
16
|
+
endpoint: "meta",
|
|
17
|
+
options,
|
|
18
|
+
selectData,
|
|
19
|
+
});
|
|
20
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { WebsiteExtractorOptions, WebsiteExtractorResponse, WebsiteExtractorResult } from "./websiteExtractorTypes.js";
|
|
2
|
+
export interface WebsiteMetaPayload {
|
|
3
|
+
title?: string | null;
|
|
4
|
+
description?: string | null;
|
|
5
|
+
language?: string | null;
|
|
6
|
+
canonicalUrl?: string | null;
|
|
7
|
+
feedUrl?: string | null;
|
|
8
|
+
textContentLength?: number | null;
|
|
9
|
+
metaTags: Record<string, string>;
|
|
10
|
+
}
|
|
11
|
+
export type WebsiteMetaResponse = WebsiteExtractorResponse<WebsiteMetaPayload>;
|
|
12
|
+
export declare function useWebsiteMetaExtractor(options: WebsiteExtractorOptions): WebsiteExtractorResult<WebsiteMetaPayload, WebsiteMetaResponse>;
|
|
@@ -0,0 +1,20 @@
|
|
|
1
|
+
import { useCallback } from "react";
|
|
2
|
+
import { useWebsiteExtractorBase } from "./useWebsiteExtractorBase.js";
|
|
3
|
+
export function useWebsiteMetaExtractor(options) {
|
|
4
|
+
const selectData = useCallback((payload) => {
|
|
5
|
+
return {
|
|
6
|
+
title: payload.title ?? undefined,
|
|
7
|
+
description: payload.description ?? undefined,
|
|
8
|
+
language: payload.language ?? undefined,
|
|
9
|
+
canonicalUrl: payload.canonicalUrl ?? undefined,
|
|
10
|
+
feedUrl: payload.feedUrl ?? null,
|
|
11
|
+
textContentLength: payload.textContentLength ?? undefined,
|
|
12
|
+
metaTags: payload.metaTags ?? {},
|
|
13
|
+
};
|
|
14
|
+
}, []);
|
|
15
|
+
return useWebsiteExtractorBase({
|
|
16
|
+
endpoint: "meta",
|
|
17
|
+
options,
|
|
18
|
+
selectData,
|
|
19
|
+
});
|
|
20
|
+
}
|
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
import { useCallback } from "react";
|
|
2
|
+
import { useWebsiteExtractorBase } from "./useWebsiteExtractorBase.js";
|
|
3
|
+
export function useWebsiteRssExtractor(options) {
|
|
4
|
+
const selectData = useCallback((payload) => {
|
|
5
|
+
return {
|
|
6
|
+
feedUrl: payload.feedUrl ?? null,
|
|
7
|
+
feeds: payload.feeds ?? [],
|
|
8
|
+
};
|
|
9
|
+
}, []);
|
|
10
|
+
return useWebsiteExtractorBase({
|
|
11
|
+
endpoint: "rss",
|
|
12
|
+
options,
|
|
13
|
+
selectData,
|
|
14
|
+
});
|
|
15
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { WebsiteExtractorOptions, WebsiteExtractorResponse, WebsiteExtractorResult } from "./websiteExtractorTypes.js";
|
|
2
|
+
export interface WebsiteRssFeed {
|
|
3
|
+
url: string;
|
|
4
|
+
feedType: string;
|
|
5
|
+
title?: string | null;
|
|
6
|
+
}
|
|
7
|
+
export interface WebsiteRssPayload {
|
|
8
|
+
feedUrl?: string | null;
|
|
9
|
+
feeds: WebsiteRssFeed[];
|
|
10
|
+
}
|
|
11
|
+
export type WebsiteRssResponse = WebsiteExtractorResponse<WebsiteRssPayload>;
|
|
12
|
+
export declare function useWebsiteRssExtractor(options: WebsiteExtractorOptions): WebsiteExtractorResult<WebsiteRssPayload, WebsiteRssResponse>;
|