groove-dev 0.27.134 → 0.27.136

package/test.py

@@ -0,0 +1,571 @@
+#!/usr/bin/python3
+import os
+from dotenv import load_dotenv
+import mysql.connector
+import logging
+import csv
+import urllib.parse
+
+import json
+from openai import OpenAI
+import time
+
+# Load environment variables
+env_path = "/var/www/html/security/.env"
+load_dotenv(dotenv_path=env_path)
+
+# Logging setup
+logging.basicConfig(
+    filename="/var/www/facebook/catalog/logs/pull-inventory-facebook.log",
+    level=logging.INFO,
+    format="%(asctime)s %(levelname)s: %(message)s"
+)
+
+# Database credentials
+DB_PASS = os.getenv('PASSWORD')
+DATABASE = os.getenv('DBNAME_CLIENT')
+client = OpenAI(api_key=os.getenv("OPENAI_API_KEY"))
+
+def get_facebook_body_style(body_field_from_vauto):
+    """
+    Translates a descriptive body field (e.g., "4D Sport Utility") 
+    into a Facebook-approved body style enum.
+    """
+    if not body_field_from_vauto:
+        return 'OTHER'
+    body = body_field_from_vauto.lower()
+    if 'convertible' in body:
+        return 'CONVERTIBLE'
+    elif 'crewmax' in body or 'crew cab' in body or 'supercrew' in body or 'quad cab' in body or 'pickup' in body or 'truck' in body or 'xtra cab' in body:
+        return 'TRUCK'
+    elif 'sport utility' in body or 'suv' in body:
+        return 'SUV'
+    elif 'coupe' in body:
+        return 'COUPE'
+    elif 'passenger van' in body or 'minivan' in body or 'van' in body:
+        return 'VAN'
+    elif 'sedan' in body:
+        return 'SEDAN'
+    elif 'hatchback' in body:
+        return 'HATCHBACK'
+    elif 'wagon' in body:
+        return 'WAGON'
+    else:
+        return 'OTHER'
+
+def connect_to_db():
+    """Connect to MySQL database."""
+    try:
+        conn = mysql.connector.connect(
+            host="localhost",
+            user="root",
+            password=DB_PASS,
+            database=DATABASE
+        )
+        return conn
+    except mysql.connector.Error as e:
+        logging.error(f"Database connection error: {e}")
+        return None
+
+# --- RENAMED: This function gets dealer *config*, not vAuto data ---
+def get_facebook_dealers():
+    """Fetch dealers with Facebook integration."""
+    conn = connect_to_db()
+    if not conn:
+        return []
+    
+    cursor = conn.cursor(dictionary=True)
+    try:
+        # This query is correct, it's fetching dealers *to process*
+        cursor.execute("SELECT id, dealer_name, search_link_new, search_link_used, address, city, state, postal_code, phone, privacy_policy_url, inventory_type, campaign_product_id, ad_prospecting_id, page_id FROM facebook_dealers WHERE process_catalog = true AND dealer_name = 'San Jose Mitsubishi'")
+        dealers = cursor.fetchall()
+        logging.info(f"Retrieved {len(dealers)} dealers for Facebook feed generation.")
+        return dealers
+    except mysql.connector.Error as e:
+        logging.error(f"Error fetching dealers: {e}")
+        return []
+    finally:
+        cursor.close()
+        conn.close()
+
+# --- NEW: Function to get inventory from dealer_inventory table ---
+def get_dealer_inventory(dealer_name):
+    """Fetch all active inventory for a specific dealer from the database."""
+    conn = connect_to_db()
+    if not conn:
+        return []
+    
+    cursor = conn.cursor(dictionary=True) # dictionary=True is key!
+    try:
+        cursor.execute("""
+            SELECT * FROM dealer_inventory 
+            WHERE dealer_name = %s AND vehicle_status = 'active'
+        """, (dealer_name,))
+        inventory = cursor.fetchall()
+        logging.info(f"Retrieved {len(inventory)} active vehicles from DB for {dealer_name}")
+        return inventory
+    except mysql.connector.Error as e:
+        logging.error(f"Error fetching inventory for {dealer_name}: {e}")
+        return []
+    finally:
+        cursor.close()
+        conn.close()
+
+
+def get_vehicle_copy(dealer_name, vin):
+    conn = connect_to_db()
+    if not conn:
+        return None
+    try:
+        cursor = conn.cursor(dictionary=True)
+        cursor.execute("""
+            SELECT * FROM facebook_vehicle_copy
+            WHERE dealer_name = %s AND vin = %s
+        """, (dealer_name, vin))
+        result = cursor.fetchone()
+        return result
+    finally:
+        cursor.close()
+        conn.close()
+
+
+def save_vehicle_copy(dealer_name, vin, copy_data):
+    conn = connect_to_db()
+    if not conn:
+        return False
+    try:
+        cursor = conn.cursor()
+        query = """
+            INSERT INTO facebook_vehicle_copy
+            (dealer_name, vin, custom_headline, custom_description,
+             custom_headline_retarget, custom_description_retarget, raw_json)
+            VALUES (%s, %s, %s, %s, %s, %s, %s)
+            ON DUPLICATE KEY UPDATE
+                custom_headline = VALUES(custom_headline),
+                custom_description = VALUES(custom_description),
+                custom_headline_retarget = VALUES(custom_headline_retarget),
+                custom_description_retarget = VALUES(custom_description_retarget),
+                raw_json = VALUES(raw_json)
+        """
+        cursor.execute(query, (
+            dealer_name, vin,
+            copy_data.get("custom_headline"),
+            copy_data.get("custom_description"),
+            copy_data.get("custom_headline_retarget"),
+            copy_data.get("custom_description_retarget"),
+            json.dumps(copy_data)
+        ))
+        conn.commit()
+        return True
+    finally:
+        cursor.close()
+        conn.close()
+
+
+# --- MODIFIED: Handles DB data types (int, Decimal) and column names ---
+def create_prospecting_copy(dealer_name, vehicle):
+    """Generate AI ad copy from a DB inventory row (dictionary)."""
+    # Normalize key names from DB and cast non-strings
+    year = str(vehicle.get('Year', ''))
+    make = (vehicle.get('Make', '') or '').strip()
+    model = (vehicle.get('Model', '') or '').strip()
+    series = (vehicle.get('Series', '') or '').strip()
+    engine = (vehicle.get('Engine', '') or '').strip()
+    days_on_lot = str(vehicle.get('Age', ''))
+    vin = (vehicle.get('VIN', '') or '').strip()
+    price = str(vehicle.get('Price', ''))
+    body = (vehicle.get('Body', '') or '').strip()
+    transmission = (vehicle.get('Transmission', '') or '').strip()
+    drivetrain = (vehicle.get('Drivetrain_Desc', '') or '').strip() # DB col: Drivetrain_Desc
+    mileage = str(vehicle.get('Odometer', ''))
+    fuel = (vehicle.get('Fuel', '') or '').strip()
+    colour = (vehicle.get('Colour', '') or '').strip()
+    features = (vehicle.get('Features', '') or '').strip()
+    description = (vehicle.get('Description', '') or '').strip()
+
+    # Handle the "New_Used" column from DB
+    vehicle_new_used = (vehicle.get('New_Used') or '').strip().upper() # DB col: New_Used
+    if vehicle_new_used == 'U':
+        vehicle_new_used = 'a Used vehicle'
+    elif vehicle_new_used == 'N':
+        vehicle_new_used = 'a Brand New vehicle'
+    else:
+        logging.warning(f"VIN {vin}: Missing or invalid 'New_Used' value — defaulting to Not Sure")
+        vehicle_new_used = 'Not Sure'
+
+    # Log what we’re passing to the AI
+    logging.info(f"{dealer_name}: Processing {year} {make} {model} ({vin}) as {vehicle_new_used}")
+
+    # Construct the AI prompt
+    prompt = f"""
+    You are an expert automotive copywriter and Facebook Ads strategist. 
+    Your task is to take raw vehicle data and transform it into a complete, high-performing ad creative package,
+    formatted as a single JSON object. Your tone should be professional, yet conversational and personal.
+    Your goal is to generate excitement and trust, encouraging potential buyers to visit the dealership.
+
+    **CRITICAL RULES & CONSTRAINTS:**
+    - FINAL OUTPUT MUST BE A SINGLE, VALID JSON OBJECT AND NOTHING ELSE.
+    - ABSOLUTELY NO EMOJIS or HASHTAGS.
+    - Adhere strictly to character limits.
+    - Focus on the vehicle’s features, condition, and dealership value.
+    - Do not discuss credit, payments, or financing (Facebook will flag it).
+    - Include price naturally within the first two sentences.
+    - Avoid claims like “lowest price” or unrealistic offers.
+    - If no price or price is 0, do not mention it anywhere in the ad.
+
+    **VEHICLE DATA FOR THIS AD:**
+    Dealer Name: {dealer_name}
+    - {year} {make} {model} {series}
+    - Body Style: {body}
+    - Transmission: {transmission}
+    - Drivetrain: {drivetrain}
+    - Mileage: {mileage}
+    - Fuel Type: {fuel}
+    - Color: {colour}
+    - Price: ${price}
+    - Key Features: {features}
+    - Dealer Description: {description}
+    - Vehicle Type: {vehicle_new_used}
+    - Days on Lot: {days_on_lot}
+
+    If it's a new vehicle, do not mention mileage.
+    If it's used, mention low mileage only when relevant.
+
+    **TASK: Generate the following components as valid JSON**
+    1. `primary_text`: Full ad copy (first sentence under 125 characters, persuasive CTA to visit {dealer_name})
+    2. `headlines`: 3 distinct headlines (each under 40 characters)
+    3. `link_descriptions`: 2 short value-driven descriptions (each under 30 characters)
+    4. `call_to_action_type`: One from [SHOP_NOW, GET_QUOTE]
+
+    **FINAL JSON OUTPUT FORMAT:**
+    {{
+        "primary_text": "...",
+        "headlines": ["...", "...", "..."],
+        "link_descriptions": ["...", "..."],
+        "call_to_action_type": "..."
+    }}
+    """
+
+    max_retries = 3
+    for attempt in range(max_retries):
+        try:
+            response = client.chat.completions.create(
+                model="gpt-5-mini", # ✅ Kept your specified model
+                messages=[
+                    {"role": "system", "content": "You are a Facebook Ads expert. Respond in clean JSON only."},
+                    {"role": "user", "content": prompt}
+                ],
+                response_format={"type": "json_object"},
+                reasoning_effort="low", # ✅ Kept your original parameters
+                store=False
+            )
+            ai_response = response.choices[0].message.content.strip()
+
+            try:
+                json.loads(ai_response)
+                return ai_response # On success, return the response
+            except json.JSONDecodeError:
+                logging.warning(f"AI for {vin} returned invalid JSON on attempt {attempt + 1}. Retrying...")
+                # Let the loop continue to the next attempt
+
+        except Exception as e:
+            logging.warning(f"AI generation for VIN {vin} failed on attempt {attempt + 1}/{max_retries}. Error: {e}")
+        
+        if attempt < max_retries - 1: # Don't sleep on the final failed attempt
+            time.sleep(2) # Wait 2 seconds before retrying
+
+    logging.error(f"AI generation for {vin} failed after all retries.")
+    return None # If all retries fail, return None
+
+
+# --- RENAMED & MODIFIED: Reads from DB data, not CSV file ---
+def transform_db_inventory_for_facebook(dealer, output_csv, inv_type, campaign_id, prospecting_ad_id):
+    """
+    Convert DB inventory data to Facebook Vehicle Catalog format and
+    write it to *output_csv*.
+    """
+    temp_output_csv = f"{output_csv}.tmp"
+    MAX_IMAGES = 20 # Facebook's limit for images
+
+    # --- NEW: Get inventory from database ---
+    inventory_data = get_dealer_inventory(dealer['dealer_name'])
+    if not inventory_data:
+        logging.warning(f"No active inventory found in DB for {dealer['dealer_name']}. Skipping feed generation.")
+        return
+
+    try:
+        # --- REMOVED: `with open(csv_file, 'r', ...)` ---
+        with open(temp_output_csv, 'w', encoding='utf-8', newline='') as outfile:
+            
+            # Define base fieldnames and add dynamic image fields
+            fieldnames = [
+                "vehicle_id", "vehicle_registration_plate", "vin", "make", "model", "year",
+                "transmission", "body_style", "fuel_type", "drivetrain", "description",
+            ]
+            # Add image fields: image[0].url, image[1].url, etc.
+            fieldnames.extend([f"image[{i}].url" for i in range(MAX_IMAGES)])
+            fieldnames.extend([
+                "mileage.value", "mileage.unit", "url", "title", "price",
+                "state_of_vehicle", "exterior_color", "address", "latitude", "longitude",
+                "trim", "interior_colour", "dealer_id", "dealer_name", "postal_code",
+                "dealer_phone", "fb_page_id", "dealer_communication_channel",
+                "dealer_privacy_policy_url"
+            ])
+
+            writer = csv.DictWriter(outfile, fieldnames=fieldnames)
+            writer.writeheader()
+
+            def force_https(url: str) -> str:
+                return url.replace("http://", "https://", 1) if url and url.startswith("http://") else url
+
+            # for testing to limit vehicles
+            # MAX_TEST_VEHICLES = 2  # limit for testing
+            # count = 0
+
+            # --- MODIFIED: Loop over DB inventory list, not CSV reader ---
+            for row in inventory_data:
+                # for testing to limit vehicles
+                # if count >= MAX_TEST_VEHICLES:
+                #     break
+                # count += 1
+
+                # --- REMOVED: 'sold' check (already done in SQL) ---
+
+                vin = (row.get("VIN") or "").strip()
+                if not vin:
+                    continue
+
+                # DB col: New_Used
+                db_type = (row.get('New_Used') or '').strip().upper()  # 'N' or 'U'
+                if inv_type == 'new' and db_type != 'N':
+                    continue
+                elif inv_type == 'used' and db_type != 'U':
+                    continue
+
+                cached_copy = get_vehicle_copy(dealer['dealer_name'], vin)
+                if cached_copy and cached_copy.get('custom_headline'):
+                    custom_headline = cached_copy['custom_headline']
+                    custom_description = cached_copy['custom_description']
+                    # Add other cached fields if needed
+                else:
+                    # Generate new copy
+                    logging.info(f"Generating new AI copy for VIN: {vin}")
+                    ai_response = create_prospecting_copy(dealer['dealer_name'], row)
+                    
+                    custom_headline = ""
+                    custom_description = ""
+                    
+                    if ai_response: # Check if the AI returned anything at all
+                        try:
+                            copy_json = json.loads(ai_response)
+                            # Use .get() for safe access to prevent KeyErrors
+                            headlines = copy_json.get('headlines', [])
+                            primary_text = copy_json.get('primary_text', '')
+
+                            if headlines and primary_text:
+                                custom_headline = headlines[0]
+                                custom_description = primary_text
+                                
+                                # Only save if the generation was successful
+                                save_vehicle_copy(dealer['dealer_name'], vin, {
+                                    "custom_headline": custom_headline,
+                                    "custom_description": custom_description,
+                                    "custom_headline_retarget": headlines[1] if len(headlines) > 1 else '',
+                                    "custom_description_retarget": copy_json.get('link_descriptions', [''])[0],
+                                    "raw": copy_json
+                                })
+                            else:
+                                logging.warning(f"AI response for {vin} was valid JSON but missing required keys (headlines/primary_text).")
+
+                        except json.JSONDecodeError:
+                            logging.error(f"Failed to decode AI JSON response for {vin}.")
+                    else:
+                        logging.error(f"AI generation failed for {vin}; no response was returned.")
+
+
+                # Price from DB is Decimal, float() handles it
+                price_val = float(row.get("Price") or 0)
+                price = f"{price_val:.2f} USD" # Format: 19995.00 USD
+
+                # Mileage from DB is int, str() handles it
+                mileage_val = str(row.get("Odometer") or '0')
+                mileage_unit = "MI" # Or "KM"
+
+                year = str(row.get('Year', '')) # Year from DB is int
+                make = row.get('Make', '')
+                model = row.get('Model', '')
+                # DB cols: Series_Detail, Series
+                trim = (row.get('Series_Detail') or row.get('Series') or '').strip()
+
+                vehicle_type = row.get('New_Used') # DB col: New_Used
+
+                if vehicle_type == 'U':
+                    vehicle_type = 'used'
+                else:
+                    vehicle_type = 'new'
+
+                # Use pre-built VDP URL from inventory, fallback to search_link
+                redirect_url = row.get('vdp_url')
+                if not redirect_url:
+                    base_link = dealer['search_link_used'] if vehicle_type == 'used' else dealer['search_link_new']
+                    redirect_url = base_link.replace('{VIN}', str(vin))
+
+                # Direct link to dealer VDP with tracking params (no redirect)
+                _sep = "&" if "?" in redirect_url else "?"
+                vehicle_url = redirect_url + _sep + urllib.parse.urlencode({
+                    "utm_source": "facebook",
+                    "utm_medium": "paid_social",
+                    "utm_campaign": "{{campaign.name}}",
+                    "utm_content": "{{adset.name}}",
+                    "utm_term": "{{ad.name}}",
+                    "ad_id": "{{ad.id}}",
+                    "adset_id": "{{adset.id}}",
+                    "campaign_id": "{{campaign.id}}",
+                    "site_source": "{{site_source_name}}",
+                    "placement": "{{placement}}",
+                })
+
+                # Title
+                title = f"{year} {make} {model} {trim}".strip()
+
+                # Address JSON object
+                address_obj = {
+                    "addr1": dealer.get('address', 'N/A'),
+                    "city": dealer.get('city', 'N/A'),
+                    "region": dealer.get('state', 'N/A'),
+                    "postal_code": dealer.get('postal_code', 'N/A'),
+                    "country": "US" # Assuming United States
+                }
+
+                # Transmission (Must be an approved enum)
+                transmission_raw = (row.get('Transmission') or '').lower()
+                if 'automatic' in transmission_raw or 'cvt' in transmission_raw:
+                    transmission = 'AUTOMATIC'
+                elif 'manual' in transmission_raw:
+                    transmission = 'MANUAL'
+                else:
+                    transmission = 'OTHER'
+                
+                # Drivetrain (Must be an approved enum)
+                # DB col: Drivetrain_Desc
+                drivetrain_raw = (row.get('Drivetrain_Desc') or '').upper()
+                if drivetrain_raw in ['AWD', '4WD', '4X4']:
+                    drivetrain = 'ALL_WHEEL_DRIVE'
+                elif drivetrain_raw == 'FWD':
+                    drivetrain = 'FRONT_WHEEL_DRIVE'
+                elif drivetrain_raw == 'RWD':
+                    drivetrain = 'REAR_WHEEL_DRIVE'
+                else:
+                    drivetrain = 'OTHER'
+
+                # Fuel Type (Must be an approved enum)
+                fuel_type_raw = (row.get('Fuel') or '').upper()
+                if fuel_type_raw == 'GASOLINE':
+                    fuel_type = 'GASOLINE'
+                elif fuel_type_raw == 'DIESEL':
+                    fuel_type = 'DIESEL'
+                elif fuel_type_raw == 'ELECTRIC':
+                    fuel_type = 'ELECTRIC'
+                elif fuel_type_raw in ('HYBRID', 'GASOLINE/HYBRID'):
+                    fuel_type = 'HYBRID_ENGINE'
+                elif fuel_type_raw == 'FLEX FUEL':
+                    fuel_type = 'FLEX'
+                else:
+                    fuel_type = 'OTHER'
+                
+                # --- IMAGE HANDLING ---
+                # DB col: Photo_Url_List
+                photos_raw = (row.get('Photo_Url_List') or '').split('|')
+                images = [force_https(p.strip()) for p in photos_raw if p and p.strip()]
+
+                # Skip if fewer than 4 usable images
+                if len(images) < 5:
+                    continue
+
+                # Write up to MAX_IMAGES valid URLs
+                image_dict = {}
+                for i, img_url in enumerate(images[:MAX_IMAGES]):
+                    image_dict[f"image[{i}].url"] = img_url
+
+
+                # --- ROW ASSEMBLY ---
+                output_row = {
+                    'vehicle_id': vin,
+                    'vin': vin,
+                    'make': make,
+                    'model': model,
+                    'year': year,
+                    'transmission': transmission,
+                    'body_style': get_facebook_body_style(row.get('Body')),
+                    'fuel_type': fuel_type,
+                    'drivetrain': drivetrain,
+                    'description': custom_description or f"Explore this {year} {make} {model} at {dealer['dealer_name']} today!",
+                    'mileage.value': mileage_val,
+                    'mileage.unit': mileage_unit,
+                    'url': vehicle_url,
+                    'title': custom_headline or title,
+                    'price': price,
+                    'state_of_vehicle': vehicle_type,
+                    'exterior_color': row.get('Colour', ''),
+                    'address': json.dumps(address_obj),
+                    'trim': trim,
+                    'interior_colour': row.get('Interior_Color', ''), # DB col: Interior_Color
+                    'dealer_id': dealer.get('id'), # Your internal dealer ID
+                    'dealer_name': dealer.get('dealer_name'),
+                    'postal_code': dealer.get('postal_code', ''),
+                    'dealer_phone': dealer.get('phone', ''),
+                    'fb_page_id': dealer.get('fb_page_id', ''), 
+                    'dealer_communication_channel': 'CHAT', # Or 'PHONE_CALL', 'LEAD_FORM'
+                    'dealer_privacy_policy_url': dealer.get('privacy_policy_url', '')
+                }
+
+                # Add the dynamic image fields to the row
+                output_row.update(image_dict)
+
+                writer.writerow(output_row)
+
+        os.replace(temp_output_csv, output_csv)
+        logging.info(f"Transformed Facebook CSV for {dealer['dealer_name']} written to {output_csv}")
+
+    except Exception as e:
+        logging.error(f"Error transforming Facebook CSV for {dealer['dealer_name']}: {e}", exc_info=True)
+        if os.path.exists(temp_output_csv):
+            os.remove(temp_output_csv)
+
+
+
+def format_filename_component(value):
+    """Formats a string for use in a filename."""
+    if value:
+        return value.replace(' ', '_').replace('&', 'and')
+    return ''
+
+# --- MODIFIED: Main function to call the new DB-driven functions ---
+def process_inventory():
+    """Main function to process inventory from DB and generate Facebook Catalog CSV."""
+    dealers = get_facebook_dealers() # Renamed function
+    if not dealers:
+        logging.info("No dealers found with Facebook integration enabled.")
+        return
+
+    for dealer in dealers:
+        dealer_name = dealer["dealer_name"]
+        dealer_id = dealer["id"]
+        inv_type = dealer['inventory_type']
+
+        campaign_id = dealer['campaign_product_id']
+        prospecting_ad_id = dealer['campaign_product_id']
+
+        # --- REMOVED: csv_file path and os.path.exists check ---
+        safe_dealer_name = format_filename_component(dealer_name)
+        # --- MODIFIED: Changed "vauto_feed" to "db_feed" for clarity ---
+        output_csv = f"/var/www/html/app.studio19.io/fb_catalog_csv/{safe_dealer_name}_db_feed_english.csv"
+
+        logging.info(f"Processing DB inventory for {dealer_name}")
+        
+        # --- MODIFIED: Call renamed function and pass correct args ---
+        transform_db_inventory_for_facebook(dealer, output_csv, inv_type, campaign_id, prospecting_ad_id)
+
+
+if __name__ == "__main__":
+    process_inventory()