@aurostack/stacks 0.1.0

package/templates/py-worker/files/_gitignore

@@ -0,0 +1,176 @@
+# ---> Python
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/
+
+.vscode/
+
+test.py
+
+input/*
+!input/.gitkeep
+
+output/*
+!output/.gitkeep
+
+logs/*
+!logs/.gitkeep

package/templates/py-worker/files/config.py

@@ -0,0 +1,89 @@
+import os
+from dataclasses import dataclass
+from dotenv import load_dotenv
+
+load_dotenv()
+
+
+@dataclass
+class ProjectPath:
+    home: str
+    input: str
+    output: str
+    logs: str
+
+
+@dataclass
+class DB:
+    host: str
+    port: int
+    user: str
+    password: str
+    name: str
+
+
+@dataclass
+class Redis:
+    host: str
+    port: int
+    user: str | None
+    password: str | None
+
+
+@dataclass
+class Env:
+    path: ProjectPath
+    db: DB
+    redis: Redis
+
+
+class Config:
+    _instance = None
+
+    def __new__(cls):
+        if cls._instance is None:
+            instance = super().__new__(cls)
+
+            home = os.getenv('HOME_PATH')
+            if not home:
+                raise ValueError("HOME_PATH environment variable is not set")
+
+            instance._env = Env(
+                path=ProjectPath(
+                    home=home,
+                    input=os.path.join(home, 'input'),
+                    output=os.path.join(home, 'output'),
+                    logs=os.path.join(home, 'logs')
+                ),
+                db=DB(
+                    host=os.getenv('DB_HOST', ''),
+                    port=int(os.getenv('DB_PORT', 0)),
+                    user=os.getenv('DB_USER', ''),
+                    password=os.getenv('DB_PASSWORD', ''),
+                    name=os.getenv('DB_NAME', '')
+                ),
+                redis=Redis(
+                    host=os.getenv('REDIS_HOST', ''),
+                    port=int(os.getenv('REDIS_PORT', 0)),
+                    user=os.getenv('REDIS_USER'),
+                    password=os.getenv('REDIS_PASSWORD')
+                )
+            )
+
+            cls._instance = instance
+
+        return cls._instance
+
+    @property
+    def env(self) -> Env:
+        return self._env
+
+    @property
+    def path(self) -> ProjectPath:
+        return self._env.path
+
+    def join(self, *args):
+        return os.path.join(self._env.path.home, *args)
+
+
+CONFIG = Config()

package/templates/py-worker/files/models/decorators.py

@@ -0,0 +1,42 @@
+def repr(*fields: str):
+    """Generates __repr__ for a model from specified fields.
+
+    Usage:
+        @repr("id", "name", "created")
+        class MyModel(Base):
+            ...
+    """
+
+    def decorator(cls):
+        def __repr__(self) -> str:
+            cls_name = type(self).__name__
+            parts = ", ".join(f"{f}={getattr(self, f)!r}" for f in fields)
+            return f"{cls_name}({parts})"
+
+        cls.__repr__ = __repr__
+        return cls
+
+    return decorator
+
+
+def to_dict(*fields: str):
+    """Adds a .to_dict() method that serializes specified fields.
+    If no fields given, serializes all column attributes.
+
+    @to_dict("id", "name", "email")
+    class User(Base):
+        ...
+
+    user.to_dict()  # {"id": 1, "name": "Alice", "email": "..."}
+    """
+
+    def decorator(cls):
+        def _to_dict(self) -> dict:
+            if fields:
+                return {f: getattr(self, f) for f in fields}
+            return {c.key: getattr(self, c.key) for c in self.__table__.columns}
+
+        cls.to_dict = _to_dict
+        return cls
+
+    return decorator

package/templates/py-worker/files/models/enums.py

@@ -0,0 +1,32 @@
+from __future__ import annotations
+
+import enum
+from sqlalchemy.dialects.postgresql import ENUM
+
+
+def e(cls: type[enum.Enum]) -> ENUM:
+    """
+    Bind a Python enum to an existing Postgres enum type.
+
+    `create_type=False` is load-bearing: the type is owned by the API's
+    migrations, and letting SQLAlchemy try to create it here would fail against
+    a real database or, worse, race the migration.
+
+    `values_callable` sends the enum *values* rather than its member names,
+    which is what the column actually stores.
+    """
+    return ENUM(
+        cls,
+        name=cls.__name__,
+        schema="public",
+        create_type=False,
+        values_callable=lambda enum_cls: [member.value for member in enum_cls]
+    )
+
+
+class Gender(enum.Enum):
+    MALE = 'male'
+    FEMALE = 'female'
+
+
+GenderEnum = e(Gender)

package/templates/py-worker/files/models/mixins.py

@@ -0,0 +1,61 @@
+from __future__ import annotations
+
+from typing_extensions import Annotated
+from sqlalchemy import DateTime, Integer, BigInteger, Text, DECIMAL, Float
+from sqlalchemy.dialects.postgresql import UUID as PgUUID
+from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column
+from datetime import datetime
+from uuid import UUID, uuid4
+from decimal import Decimal
+
+
+int_pk = Annotated[int, mapped_column(
+    Integer, primary_key=True, autoincrement=True)]
+bigint_pk = Annotated[int, mapped_column(
+    BigInteger, primary_key=True, autoincrement=True)]
+text_pk = Annotated[str, mapped_column(Text, primary_key=True)]
+uuid_pk = Annotated[UUID, mapped_column(
+    PgUUID(as_uuid=True), primary_key=True, default=uuid4
+)]
+dcml = Annotated[Decimal, mapped_column(DECIMAL(18, 4))]
+dcml_nl = Annotated[
+    Decimal | None,
+    mapped_column(DECIMAL(18, 4), nullable=True)
+]
+dcml_20 = Annotated[Decimal, mapped_column(DECIMAL(20, 4))]
+dcml_20_nl = Annotated[
+    Decimal | None,
+    mapped_column(DECIMAL(20, 4), nullable=True)
+]
+flt = Annotated[float, mapped_column(Float)]
+flt_nl = Annotated[float | None, mapped_column(Float, nullable=True)]
+
+
+class Base(DeclarativeBase):
+    """Base for all ORM models."""
+
+    pass
+
+
+class CreatedAtMixin:
+    """Adds a `created_at` column that auto-updates on creation."""
+
+    created_at: Mapped[datetime] = mapped_column(
+        DateTime, default=datetime.now)
+
+
+class UpdatedAtMixin:
+    """Adds an `updated_at` column that auto-updates on modification."""
+
+    updated_at: Mapped[datetime | None] = mapped_column(
+        DateTime,
+        default=datetime.now,
+        onupdate=datetime.now,
+        nullable=True,
+    )
+
+
+class TimestampMixin(CreatedAtMixin, UpdatedAtMixin):
+    """Combines created_at + updated_at."""
+
+    pass

package/templates/py-worker/files/requirements.txt

@@ -0,0 +1,18 @@
+asyncio==4.0.0
+bullmq==2.25.2
+certifi==2026.5.20
+charset-normalizer==3.4.7
+cuid2==2.0.1
+dotenv==0.9.9
+greenlet==3.5.1
+idna==3.18
+msgpack==1.1.2
+psycopg2-binary==2.9.12
+python-dotenv==1.2.2
+redis==6.4.0
+requests==2.34.2
+semver==3.0.4
+SQLAlchemy==2.0.50
+typing_extensions==4.15.0
+urllib3==2.7.0
+websockets==16.0

package/templates/py-worker/files/utils/db.py

@@ -0,0 +1,67 @@
+"""
+@File: db.py
+@Version: 1.0
+
+Database wrapper to manage PostgreSQL connections using SQLAlchemy
+
+DB('my_database') will create a connection to the specified PostgreSQL database
+using credentials from the .env file (user, password, host, port).
+
+Usage example:
+
+from utils import DB
+
+db = DB('my_database')
+
+# Read data
+df = db.read_sql('SELECT * FROM my_table')
+
+# Write data
+db.write_sql(df, 'my_table', if_exists='append')
+
+# Use engine or session without having to create a new connection or variable
+db.session.query(...)
+db.engine.execute(...)
+# ... perform operations ...
+db.session.commit()
+
+# Close the connection when done
+db.close()
+
+"""
+
+from sqlalchemy import create_engine
+from sqlalchemy.engine import Connection
+from sqlalchemy.orm import sessionmaker, Session
+
+from config import CONFIG
+
+
+class DB:
+    def __init__(self):
+        db = CONFIG.env.db
+        self._engine = create_engine(
+            f"postgresql://{db.user}:{db.password}@{db.host}:{db.port}/{db.name}"
+        ).connect()
+        Session = sessionmaker(bind=self.engine)
+        self._session = Session()
+
+    @property
+    def engine(self) -> Connection:
+        return self._engine
+
+    @property
+    def session(self) -> Session:
+        return self._session
+
+    def close(self, session: Session | None = None):
+        """
+        Closes the given session and the engine if it is not already closed.
+
+        :param session: Pass a session object that needs to be closed. If a session object is provided, the method will close that session
+        :type session: Session | None
+        """
+        session_to_close = session if session else self.session
+        if not self.engine.closed:
+            session_to_close.close()
+            self.engine.close()

package/templates/py-worker/files/utils/logger.py

@@ -0,0 +1,97 @@
+"""
+@File: logger.py
+@Version: 1.1
+
+Logger wrapper to manage logs in one place with log rotate
+
+Logger('daily') will create daily.log file in the logs folder
+Logger('my.daily') will create my.daily.log file in the logs folder
+
+If it is necessary to store the log files in the separate folder set the second parameter 'module'
+
+Logger('daily', 'criteo') will create daily.log file in the logs/criteo folder
+
+Usage example:
+
+from utils.logger import Logger
+
+logger = Logger('my.daily')
+
+logger.info('Logging info entry.')
+logger.debug('Logging debug entry.')
+logger.warning('Logging warning entry.')
+logger.critical('Logging critical entry.')
+logger.error('Logging error entry.')
+
+If needed to print the trace info about some exception, just add second parameter 'exc_info' to the function like so:
+
+try:
+    ....
+except Exception as e:
+    logger.error('Error in the code', exc_info=e)
+
+"""
+
+import logging
+import os
+import pathlib
+from logging import handlers
+
+from config import CONFIG
+
+
+class Logger(object):
+
+    def __init__(self, name, module=None):
+        directory = f"{CONFIG.path.logs}"
+        logger_name = name
+
+        if module is not None:
+            directory = f"{directory}/{module}"
+            logger_name = f"{module}_{name}"
+            pathlib.Path(directory).mkdir(parents=True, exist_ok=True)
+
+        filename = f"{directory}/{name}.log"
+        formatter = "%(asctime)s %(levelname)-10s %(message)s"
+
+        self.logger = logging.getLogger(logger_name)
+        self.logger.setLevel(logging.DEBUG)
+
+        # make sure to add RotatingFileHandler only once
+        # you might need to update condition if you have more than 1 handler
+        if not self.logger.handlers:
+            handler = handlers.RotatingFileHandler(
+                filename,
+                mode='a',
+                maxBytes=100000,
+                backupCount=5,
+                encoding=None,
+                delay=False
+            )
+            handler.setLevel(logging.DEBUG)
+            formatter = logging.Formatter(
+                formatter, datefmt='%Y-%m-%d %H:%M:%S')
+            handler.setFormatter(formatter)
+            self.logger.addHandler(handler)
+
+    def debug(self, message, exc_info=None):
+        self.__call(self.logger.debug, message, exc_info)
+
+    def info(self, message, exc_info=None):
+        self.__call(self.logger.info, message, exc_info)
+
+    def warning(self, message, exc_info=None):
+        self.__call(self.logger.warning, message, exc_info)
+
+    def error(self, message, exc_info=None):
+        self.__call(self.logger.error, message, exc_info)
+
+    def critical(self, message, exc_info=None):
+        self.__call(self.logger.critical, message, exc_info)
+
+    @staticmethod
+    def __call(method, message, exc_info=None):
+        if exc_info is not None:
+            method(message, exc_info=exc_info)
+        else:
+            method(message)

package/templates/py-worker/files/utils/misc.py

@@ -0,0 +1,45 @@
+import os
+import pathlib
+from urllib.parse import urlparse, quote, urlencode
+from datetime import datetime, timedelta
+from typing import Callable
+
+
+# A function that takes a page number and returns
+# whether the loop should continue.
+LoopFunction = Callable[[int], bool]
+
+
+def loop(fn: LoopFunction) -> None:
+    """
+    Repeatedly calls `fn(page)` with an increasing page number starting at 1
+    until `fn` returns False.
+
+    Args:
+        fn: A callable that accepts the current page (int) and returns:
+            - True to continue to the next page
+            - False to stop looping
+    """
+    page = 1
+    while page > 0:
+        can_proceed = fn(page)
+        if not can_proceed:
+            page = 0
+        else:
+            page += 1
+
+
+def get_offset(page: int, limit: int = 10) -> int:
+    """
+    Calculate the offset for pagination.
+
+    Page is treated as 1-indexed (page=1 => offset=0).
+
+    Args:
+        page: Current page number (1-indexed).
+        limit: Number of items per page. Defaults to 10.
+
+    Returns:
+        The offset (number of items to skip).
+    """
+    return (page - 1) * limit

package/templates/py-worker/files/utils/worker.py

@@ -0,0 +1,80 @@
+from typing import Callable
+from bullmq import Worker, Job
+from urllib.parse import urlparse, quote, urlencode
+
+from config import CONFIG, Redis
+from utils.db import DB
+from utils.logger import Logger
+
+
+def build_redis_url(options: Redis):
+    # Treat empty strings like missing values
+    user = options.user or None
+    password = options.password or None
+    host = options.host
+    port = options.port
+
+    if user and password:
+        auth = f"{quote(user, safe='')}:{quote(password, safe='')}@"
+    elif user:
+        auth = f"{quote(user, safe='')}@"
+    elif password:
+        # Password-only auth keeps empty username before colon
+        auth = f":{quote(password, safe='')}@"
+    else:
+        auth = ""
+
+    return f"redis://{auth}{host}:{port}"
+
+
+def create_worker(queue_name: str, process_function: Callable, concurrency: int = 1) -> Worker:
+    """
+    Create a BullMQ Worker connected to the specified Redis instance.
+
+    Args:
+        queue_name: Name of the queue to connect to.
+        process_function: The function to process jobs.
+
+    Returns:
+        An instance of BullMQ Worker.
+    """
+    connection = build_redis_url(CONFIG.env.redis)
+    worker = Worker(
+        queue_name,
+        process_function,
+        {
+            "connection": connection,
+            "autorun": True,
+            "concurrency": concurrency
+        })
+    return worker
+
+
+def create_logger(name: str) -> Logger:
+    """
+    Create a Logger instance.
+
+    Args:
+        name: The name of the logger (and log file).
+
+    Returns:
+        An instance of Logger for a worker.
+    """
+    return Logger(name, "workers")
+
+
+class JobUtil:
+    def __init__(self, job: Job, logger: Logger):
+        self._job = job
+        self._logger = logger
+
+    async def process_log(self, message: str, progress: int | None = None):
+        if progress is not None:
+            await self._job.updateProgress(progress)
+        await self._job.log(message)
+
+    async def finalize(self, db: DB | None = None):
+        if db:
+            db.close()
+        await self._job.updateProgress(100)
+        self._logger.info(f"Job[{self._job.id}]: completed")