@finch_ren/x-scraper 0.1.0

package/dist/src/openapi/src/runtime.js

@@ -0,0 +1,339 @@
+"use strict";
+/* tslint:disable */
+/* eslint-disable */
+/**
+ * Twitter OpenAPI
+ * Twitter OpenAPI(Swagger) specification
+ *
+ * The version of the OpenAPI document: 0.0.1
+ * Contact: yuki@yuki0311.com
+ *
+ * NOTE: This class is auto generated by OpenAPI Generator (https://openapi-generator.tech).
+ * https://openapi-generator.tech
+ * Do not edit the class manually.
+ */
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TextApiResponse = exports.BlobApiResponse = exports.VoidApiResponse = exports.JSONApiResponse = exports.COLLECTION_FORMATS = exports.RequiredError = exports.FetchError = exports.ResponseError = exports.BaseAPI = exports.DefaultConfig = exports.Configuration = exports.BASE_PATH = void 0;
+exports.querystring = querystring;
+exports.exists = exists;
+exports.mapValues = mapValues;
+exports.canConsumeForm = canConsumeForm;
+exports.BASE_PATH = "https://x.com/i/api".replace(/\/+$/, "");
+class Configuration {
+    constructor(configuration = {}) {
+        this.configuration = configuration;
+    }
+    set config(configuration) {
+        this.configuration = configuration;
+    }
+    get basePath() {
+        return this.configuration.basePath != null ? this.configuration.basePath : exports.BASE_PATH;
+    }
+    get fetchApi() {
+        return this.configuration.fetchApi;
+    }
+    get middleware() {
+        return this.configuration.middleware || [];
+    }
+    get queryParamsStringify() {
+        return this.configuration.queryParamsStringify || querystring;
+    }
+    get username() {
+        return this.configuration.username;
+    }
+    get password() {
+        return this.configuration.password;
+    }
+    get apiKey() {
+        const apiKey = this.configuration.apiKey;
+        if (apiKey) {
+            return typeof apiKey === 'function' ? apiKey : () => apiKey;
+        }
+        return undefined;
+    }
+    get accessToken() {
+        const accessToken = this.configuration.accessToken;
+        if (accessToken) {
+            return typeof accessToken === 'function' ? accessToken : async () => accessToken;
+        }
+        return undefined;
+    }
+    get headers() {
+        return this.configuration.headers;
+    }
+    get credentials() {
+        return this.configuration.credentials;
+    }
+}
+exports.Configuration = Configuration;
+exports.DefaultConfig = new Configuration();
+/**
+ * This is the base class for all generated API classes.
+ */
+class BaseAPI {
+    constructor(configuration = exports.DefaultConfig) {
+        this.configuration = configuration;
+        this.fetchApi = async (url, init) => {
+            let fetchParams = { url, init };
+            for (const middleware of this.middleware) {
+                if (middleware.pre) {
+                    fetchParams = await middleware.pre({
+                        fetch: this.fetchApi,
+                        ...fetchParams,
+                    }) || fetchParams;
+                }
+            }
+            let response = undefined;
+            try {
+                response = await (this.configuration.fetchApi || fetch)(fetchParams.url, fetchParams.init);
+            }
+            catch (e) {
+                for (const middleware of this.middleware) {
+                    if (middleware.onError) {
+                        response = await middleware.onError({
+                            fetch: this.fetchApi,
+                            url: fetchParams.url,
+                            init: fetchParams.init,
+                            error: e,
+                            response: response ? response.clone() : undefined,
+                        }) || response;
+                    }
+                }
+                if (response === undefined) {
+                    if (e instanceof Error) {
+                        throw new FetchError(e, 'The request failed and the interceptors did not return an alternative response');
+                    }
+                    else {
+                        throw e;
+                    }
+                }
+            }
+            for (const middleware of this.middleware) {
+                if (middleware.post) {
+                    response = await middleware.post({
+                        fetch: this.fetchApi,
+                        url: fetchParams.url,
+                        init: fetchParams.init,
+                        response: response.clone(),
+                    }) || response;
+                }
+            }
+            return response;
+        };
+        this.middleware = configuration.middleware;
+    }
+    withMiddleware(...middlewares) {
+        const next = this.clone();
+        next.middleware = next.middleware.concat(...middlewares);
+        return next;
+    }
+    withPreMiddleware(...preMiddlewares) {
+        const middlewares = preMiddlewares.map((pre) => ({ pre }));
+        return this.withMiddleware(...middlewares);
+    }
+    withPostMiddleware(...postMiddlewares) {
+        const middlewares = postMiddlewares.map((post) => ({ post }));
+        return this.withMiddleware(...middlewares);
+    }
+    /**
+     * Check if the given MIME is a JSON MIME.
+     * JSON MIME examples:
+     *   application/json
+     *   application/json; charset=UTF8
+     *   APPLICATION/JSON
+     *   application/vnd.company+json
+     * @param mime - MIME (Multipurpose Internet Mail Extensions)
+     * @return True if the given MIME is JSON, false otherwise.
+     */
+    isJsonMime(mime) {
+        if (!mime) {
+            return false;
+        }
+        return BaseAPI.jsonRegex.test(mime);
+    }
+    async request(context, initOverrides) {
+        const { url, init } = await this.createFetchParams(context, initOverrides);
+        const response = await this.fetchApi(url, init);
+        if (response && (response.status >= 200 && response.status < 300)) {
+            return response;
+        }
+        throw new ResponseError(response, 'Response returned an error code');
+    }
+    async createFetchParams(context, initOverrides) {
+        let url = this.configuration.basePath + context.path;
+        if (context.query !== undefined && Object.keys(context.query).length !== 0) {
+            // only add the querystring to the URL if there are query parameters.
+            // this is done to avoid urls ending with a "?" character which buggy webservers
+            // do not handle correctly sometimes.
+            url += '?' + this.configuration.queryParamsStringify(context.query);
+        }
+        const headers = Object.assign({}, this.configuration.headers, context.headers);
+        Object.keys(headers).forEach(key => headers[key] === undefined ? delete headers[key] : {});
+        const initOverrideFn = typeof initOverrides === "function"
+            ? initOverrides
+            : async () => initOverrides;
+        const initParams = {
+            method: context.method,
+            headers,
+            body: context.body,
+            credentials: this.configuration.credentials,
+        };
+        const overriddenInit = {
+            ...initParams,
+            ...(await initOverrideFn({
+                init: initParams,
+                context,
+            }))
+        };
+        let body;
+        if (isFormData(overriddenInit.body)
+            || (overriddenInit.body instanceof URLSearchParams)
+            || isBlob(overriddenInit.body)) {
+            body = overriddenInit.body;
+        }
+        else if (this.isJsonMime(headers['Content-Type'])) {
+            body = JSON.stringify(overriddenInit.body);
+        }
+        else {
+            body = overriddenInit.body;
+        }
+        const init = {
+            ...overriddenInit,
+            body
+        };
+        return { url, init };
+    }
+    /**
+     * Create a shallow clone of `this` by constructing a new instance
+     * and then shallow cloning data members.
+     */
+    clone() {
+        const constructor = this.constructor;
+        const next = new constructor(this.configuration);
+        next.middleware = this.middleware.slice();
+        return next;
+    }
+}
+exports.BaseAPI = BaseAPI;
+BaseAPI.jsonRegex = new RegExp('^(:?application\/json|[^;/ \t]+\/[^;/ \t]+[+]json)[ \t]*(:?;.*)?$', 'i');
+;
+function isBlob(value) {
+    return typeof Blob !== 'undefined' && value instanceof Blob;
+}
+function isFormData(value) {
+    return typeof FormData !== "undefined" && value instanceof FormData;
+}
+class ResponseError extends Error {
+    constructor(response, msg) {
+        super(msg);
+        this.response = response;
+        this.name = "ResponseError";
+    }
+}
+exports.ResponseError = ResponseError;
+class FetchError extends Error {
+    constructor(cause, msg) {
+        super(msg);
+        this.cause = cause;
+        this.name = "FetchError";
+    }
+}
+exports.FetchError = FetchError;
+class RequiredError extends Error {
+    constructor(field, msg) {
+        super(msg);
+        this.field = field;
+        this.name = "RequiredError";
+    }
+}
+exports.RequiredError = RequiredError;
+exports.COLLECTION_FORMATS = {
+    csv: ",",
+    ssv: " ",
+    tsv: "\t",
+    pipes: "|",
+};
+function querystring(params, prefix = '') {
+    return Object.keys(params)
+        .map(key => querystringSingleKey(key, params[key], prefix))
+        .filter(part => part.length > 0)
+        .join('&');
+}
+function querystringSingleKey(key, value, keyPrefix = '') {
+    const fullKey = keyPrefix + (keyPrefix.length ? `[${key}]` : key);
+    if (value instanceof Array) {
+        const multiValue = value.map(singleValue => encodeURIComponent(String(singleValue)))
+            .join(`&${encodeURIComponent(fullKey)}=`);
+        return `${encodeURIComponent(fullKey)}=${multiValue}`;
+    }
+    if (value instanceof Set) {
+        const valueAsArray = Array.from(value);
+        return querystringSingleKey(key, valueAsArray, keyPrefix);
+    }
+    if (value instanceof Date) {
+        return `${encodeURIComponent(fullKey)}=${encodeURIComponent(value.toISOString())}`;
+    }
+    if (value instanceof Object) {
+        return querystring(value, fullKey);
+    }
+    return `${encodeURIComponent(fullKey)}=${encodeURIComponent(String(value))}`;
+}
+function exists(json, key) {
+    const value = json[key];
+    return value !== null && value !== undefined;
+}
+function mapValues(data, fn) {
+    const result = {};
+    for (const key of Object.keys(data)) {
+        result[key] = fn(data[key]);
+    }
+    return result;
+}
+function canConsumeForm(consumes) {
+    for (const consume of consumes) {
+        if ('multipart/form-data' === consume.contentType) {
+            return true;
+        }
+    }
+    return false;
+}
+class JSONApiResponse {
+    constructor(raw, transformer = (jsonValue) => jsonValue) {
+        this.raw = raw;
+        this.transformer = transformer;
+    }
+    async value() {
+        return this.transformer(await this.raw.json());
+    }
+}
+exports.JSONApiResponse = JSONApiResponse;
+class VoidApiResponse {
+    constructor(raw) {
+        this.raw = raw;
+    }
+    async value() {
+        return undefined;
+    }
+}
+exports.VoidApiResponse = VoidApiResponse;
+class BlobApiResponse {
+    constructor(raw) {
+        this.raw = raw;
+    }
+    async value() {
+        return await this.raw.blob();
+    }
+    ;
+}
+exports.BlobApiResponse = BlobApiResponse;
+class TextApiResponse {
+    constructor(raw) {
+        this.raw = raw;
+    }
+    async value() {
+        return await this.raw.text();
+    }
+    ;
+}
+exports.TextApiResponse = TextApiResponse;
+//# sourceMappingURL=runtime.js.map

package/dist/src/utils/api.d.ts

@@ -0,0 +1,30 @@
+import { ApiUtilsHeader, CursorApiUtilsResponse, TweetApiUtilsData, UserApiUtilsData } from '../models';
+import * as i from '../openapi';
+export type InitOverridesType = (flag: {
+    [key: string]: any;
+}) => i.InitOverrideFunction;
+export declare const getKwargs: (flag: {
+    [key: string]: any;
+}, additional: {
+    [key: string]: any;
+}) => any;
+export declare const errorCheck: <T>(data: T | undefined, error: i.ErrorResponse[] | undefined) => T;
+export declare const instructionToEntry: (item: i.InstructionUnion[]) => i.TimelineAddEntry[];
+export declare const instructionConverter: (item: i.InstructionUnion[]) => TweetApiUtilsData[];
+export declare const tweetEntriesConverter: (item: i.TimelineAddEntry[]) => TweetApiUtilsData[];
+export declare const moduleConverter: (item: i.ModuleItem[]) => TweetApiUtilsData[];
+type buildTweetApiUtilsArgs = {
+    result: i.ItemResult;
+    promotedMetadata?: any;
+    reply?: i.TimelineTweet[];
+};
+export declare const buildTweetApiUtils: (args: buildTweetApiUtilsArgs) => TweetApiUtilsData | undefined;
+export declare const tweetResultsConverter: (tweetResults: i.ItemResult) => i.Tweet | undefined;
+export declare const userOrNullConverter: (userResults: i.UserUnion) => i.User | undefined;
+export declare const userEntriesConverter: (item: i.TimelineAddEntry[]) => i.UserResults[];
+export declare const userResultConverter: (user: i.UserResults[]) => UserApiUtilsData[];
+export declare const entriesCursor: (item: i.TimelineAddEntry[]) => CursorApiUtilsResponse;
+export declare const buildCursor: (cursorList: i.TimelineTimelineCursor[]) => CursorApiUtilsResponse;
+export declare const buildHeader: (headers: Headers) => ApiUtilsHeader;
+export {};
+//# sourceMappingURL=api.d.ts.map

package/dist/src/utils/api.js

@@ -0,0 +1,270 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.buildHeader = exports.buildCursor = exports.entriesCursor = exports.userResultConverter = exports.userEntriesConverter = exports.userOrNullConverter = exports.tweetResultsConverter = exports.buildTweetApiUtils = exports.moduleConverter = exports.tweetEntriesConverter = exports.instructionConverter = exports.instructionToEntry = exports.errorCheck = exports.getKwargs = void 0;
+const i = __importStar(require("../openapi"));
+const getKwargs = (flag, additional) => {
+    const kwargs = { pathQueryId: flag.queryId };
+    if (flag.variables != undefined) {
+        kwargs.variables = JSON.stringify({ ...flag.variables, ...additional });
+    }
+    if (flag.features != undefined) {
+        kwargs.features = JSON.stringify(flag.features);
+    }
+    if (flag.fieldToggles != undefined) {
+        kwargs.fieldToggles = JSON.stringify(flag.fieldToggles);
+    }
+    return kwargs;
+};
+exports.getKwargs = getKwargs;
+const errorCheck = (data, error) => {
+    if (data == undefined) {
+        throw Error(error?.map((e) => e.message).join(', ') ?? 'No data');
+    }
+    return data;
+};
+exports.errorCheck = errorCheck;
+const instructionToEntry = (item) => {
+    return item.flatMap((e) => {
+        if (e.type == i.InstructionType.TimelineAddEntries) {
+            return e.entries;
+        }
+        else if (e.type == i.InstructionType.TimelineAddToModule) {
+            return [];
+        }
+        else if (e.type == i.InstructionType.TimelineReplaceEntry) {
+            return [e.entry];
+        }
+        return [];
+    });
+};
+exports.instructionToEntry = instructionToEntry;
+const instructionConverter = (item) => {
+    return item
+        .flatMap((e) => {
+        if (e.type == i.InstructionType.TimelineAddEntries) {
+            return [];
+        }
+        else if (e.type == i.InstructionType.TimelineAddToModule) {
+            const item = e.moduleItems ?? [];
+            return (0, exports.moduleConverter)(item);
+        }
+        else if (e.type == i.InstructionType.TimelineReplaceEntry) {
+            return [];
+        }
+        return [];
+    })
+        .filter((e) => e != undefined)
+        .flat();
+};
+exports.instructionConverter = instructionConverter;
+const tweetEntriesConverter = (item) => {
+    return item
+        .map((e) => {
+        if (e.content.entryType == i.ContentEntryType.TimelineTimelineItem) {
+            const item = e.content.itemContent;
+            const timeline = item.itemType == i.ContentItemType.TimelineTweet ? item : undefined;
+            if (timeline == undefined)
+                return undefined;
+            return [
+                (0, exports.buildTweetApiUtils)({
+                    result: timeline.tweetResults,
+                    promotedMetadata: timeline.promotedMetadata,
+                }),
+            ];
+        }
+        else if (e.content.entryType == i.ContentEntryType.TimelineTimelineModule) {
+            const item = e.content.items ?? [];
+            return (0, exports.moduleConverter)(item);
+        }
+    })
+        .filter((e) => e != undefined)
+        .map((e) => e.filter((e) => e != undefined))
+        .flat();
+};
+exports.tweetEntriesConverter = tweetEntriesConverter;
+const moduleConverter = (item) => {
+    const timelineList = item
+        .filter((e) => e.item.itemContent.itemType == i.ContentItemType.TimelineTweet)
+        .map((e) => e.item.itemContent);
+    if (timelineList.length == 0)
+        return [];
+    if (timelineList[0].tweetDisplayType == i.TimelineTweetTweetDisplayTypeEnum.MediaGrid) {
+        return timelineList
+            .map((e) => (0, exports.buildTweetApiUtils)({
+            result: e.tweetResults,
+            promotedMetadata: e.promotedMetadata,
+        }))
+            .filter((e) => e != undefined);
+    }
+    else {
+        const timeline = timelineList[0];
+        return [
+            (0, exports.buildTweetApiUtils)({
+                result: timeline.tweetResults,
+                promotedMetadata: timeline.promotedMetadata,
+                reply: timelineList.slice(1),
+            }),
+        ].filter((e) => e != undefined);
+    }
+};
+exports.moduleConverter = moduleConverter;
+const buildTweetApiUtils = (args) => {
+    const tweet = (0, exports.tweetResultsConverter)(args.result);
+    if (tweet == undefined)
+        return undefined;
+    const result = tweet.core?.userResults.result;
+    const user = result && (0, exports.userOrNullConverter)(result);
+    if (user == undefined)
+        return undefined;
+    const quoted = tweet.quotedStatusResult;
+    const retweeted = tweet.legacy?.retweetedStatusResult;
+    const reply = args.reply
+        ?.map((e) => (0, exports.buildTweetApiUtils)({ result: e.tweetResults, promotedMetadata: e.promotedMetadata }))
+        .filter((e) => e != undefined) ?? [];
+    return {
+        raw: args.result,
+        promotedMetadata: args.promotedMetadata,
+        tweet: tweet,
+        user: user,
+        replies: reply,
+        retweeted: retweeted == undefined ? undefined : (0, exports.buildTweetApiUtils)({ result: retweeted }),
+        quoted: quoted == undefined ? undefined : (0, exports.buildTweetApiUtils)({ result: quoted }),
+    };
+};
+exports.buildTweetApiUtils = buildTweetApiUtils;
+const tweetResultsConverter = (tweetResults) => {
+    if (tweetResults.result == undefined)
+        return undefined;
+    switch (tweetResults.result.typename) {
+        case i.TypeName.Tweet:
+            return tweetResults.result;
+        case i.TypeName.TweetWithVisibilityResults:
+            return tweetResults.result.tweet;
+        case i.TypeName.TweetTombstone:
+            return undefined;
+    }
+};
+exports.tweetResultsConverter = tweetResultsConverter;
+const userOrNullConverter = (userResults) => {
+    let user;
+    if (userResults.typename == i.TypeName.User) {
+        user = userResults;
+    }
+    else if (userResults.typename == undefined) {
+        user = userResults; // error fallback
+    }
+    if (user) {
+        if (user.core && user.legacy) {
+            if (!user.legacy.screenName && user.core.screenName) {
+                user.legacy.screenName = user.core.screenName;
+            }
+            if (!user.legacy.name && user.core.name) {
+                user.legacy.name = user.core.name;
+            }
+            if (!user.legacy.createdAt && user.core.createdAt) {
+                user.legacy.createdAt = user.core.createdAt;
+            }
+        }
+        return user;
+    }
+};
+exports.userOrNullConverter = userOrNullConverter;
+const userEntriesConverter = (item) => {
+    return item
+        .map((e) => {
+        if (e.content.typename == i.TypeName.TimelineTimelineItem) {
+            const item = e.content.itemContent;
+            if (item.itemType == i.ContentItemType.TimelineUser) {
+                return item.userResults;
+            }
+        }
+    })
+        .filter((e) => e != undefined);
+};
+exports.userEntriesConverter = userEntriesConverter;
+const userResultConverter = (user) => {
+    return user
+        .map((e) => {
+        const user = e.result && (0, exports.userOrNullConverter)(e.result);
+        if (user == undefined)
+            return undefined;
+        return { raw: e, user: user };
+    })
+        .filter((e) => e != undefined);
+};
+exports.userResultConverter = userResultConverter;
+const entriesCursor = (item) => {
+    const cursorList = item
+        .map((e) => {
+        if (e.content.entryType == i.ContentEntryType.TimelineTimelineCursor) {
+            return e.content;
+        }
+        else if (e.content.entryType == i.ContentEntryType.TimelineTimelineItem) {
+            const item = e.content;
+            if (item.itemContent.itemType == i.ContentItemType.TimelineTimelineCursor) {
+                return item.itemContent;
+            }
+        }
+    })
+        .filter((e) => e != undefined);
+    return (0, exports.buildCursor)(cursorList);
+};
+exports.entriesCursor = entriesCursor;
+const buildCursor = (cursorList) => {
+    return {
+        top: cursorList.find((e) => e.cursorType == i.CursorType.Top),
+        bottom: cursorList.find((e) => e.cursorType == i.CursorType.Bottom),
+    };
+};
+exports.buildCursor = buildCursor;
+const buildHeader = (headers) => {
+    return {
+        raw: headers,
+        connectionHash: headers.get('x-connection-hash'),
+        contentTypeOptions: headers.get('x-content-type-options'),
+        frameOptions: headers.get('x-frame-options'),
+        rateLimitLimit: Number(headers.get('x-rate-limit-limit') ?? 0),
+        rateLimitRemaining: Number(headers.get('x-rate-limit-remaining') ?? 0),
+        rateLimitReset: Number(headers.get('x-rate-limit-reset') ?? 0),
+        responseTime: Number(headers.get('x-response-time')),
+        tfePreserveBody: headers.get('x-tfe-preserve-body') == 'true',
+        transactionId: headers.get('x-transaction-id'),
+        twitterResponseTags: headers.get('x-twitter-response-tags'),
+        xssProtection: Number(headers.get('x-xss-protection')),
+    };
+};
+exports.buildHeader = buildHeader;
+//# sourceMappingURL=api.js.map

package/dist/src/utils/index.d.ts

@@ -0,0 +1,2 @@
+export * from './api';
+//# sourceMappingURL=index.d.ts.map

package/dist/src/utils/index.js

@@ -0,0 +1,18 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __exportStar = (this && this.__exportStar) || function(m, exports) {
+    for (var p in m) if (p !== "default" && !Object.prototype.hasOwnProperty.call(exports, p)) __createBinding(exports, m, p);
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+__exportStar(require("./api"), exports);
+//# sourceMappingURL=index.js.map